ofz#45878 add a way to set some limit on hugely complex clips

Change-Id: I6bbf7c6068560e3bb656560fb5c6cc2ed72cecd4
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/131907
Tested-by: Jenkins
Reviewed-by: Caolán McNamara <caolanm@redhat.com>

7 files changed, 54 insertions, 17 deletions

diff --git a/basegfx/source/polygon/b2dpolygonclipper.cxx b/basegfx/source/polygon/b2dpolygonclipper.cxx
index 246d5a10ab84..69eba2c84fa7 100644
--- a/basegfx/source/polygon/b2dpolygonclipper.cxx
+++ b/basegfx/source/polygon/b2dpolygonclipper.cxx
@@ -25,6 +25,7 @@
 #include <basegfx/polygon/b2dpolypolygontools.hxx>
 #include <basegfx/curve/b2dcubicbezier.hxx>
 #include <basegfx/utils/rectcliptools.hxx>
+#include <sal/log.hxx>
 
 namespace basegfx::utils
 {
@@ -330,7 +331,8 @@ namespace basegfx::utils
             return aRetval;
         }
 
-        B2DPolyPolygon clipPolyPolygonOnPolyPolygon(const B2DPolyPolygon& rCandidate, const B2DPolyPolygon& rClip, bool bInside, bool bStroke)
+        B2DPolyPolygon clipPolyPolygonOnPolyPolygon(const B2DPolyPolygon& rCandidate, const B2DPolyPolygon& rClip,
+                                                    bool bInside, bool bStroke, size_t* pPointLimit)
         {
             B2DPolyPolygon aRetval;
 
@@ -471,7 +473,14 @@ namespace basegfx::utils
 
 
                     // prepare 2nd source polygon in same way
-                    B2DPolyPolygon aMergePolyPolygonB = solveCrossovers(rCandidate);
+                    B2DPolyPolygon aMergePolyPolygonB = solveCrossovers(rCandidate, pPointLimit);
+
+                    if (pPointLimit && !*pPointLimit)
+                    {
+                        SAL_WARN("basegfx", "clipPolyPolygonOnPolyPolygon hit point limit");
+                        return aRetval;
+                    }
+
                     aMergePolyPolygonB = stripNeutralPolygons(aMergePolyPolygonB);
                     aMergePolyPolygonB = correctOrientations(aMergePolyPolygonB);
 
diff --git a/basegfx/source/polygon/b2dpolygoncutandtouch.cxx b/basegfx/source/polygon/b2dpolygoncutandtouch.cxx
index 99a73ca82bc3..d5ab5887da61 100644
--- a/basegfx/source/polygon/b2dpolygoncutandtouch.cxx
+++ b/basegfx/source/polygon/b2dpolygoncutandtouch.cxx
@@ -19,6 +19,7 @@
 
 #include <basegfx/polygon/b2dpolygoncutandtouch.hxx>
 #include <osl/diagnose.h>
+#include <sal/log.hxx>
 #include <basegfx/numeric/ftools.hxx>
 #include <basegfx/point/b2dpoint.hxx>
 #include <basegfx/vector/b2dvector.hxx>
@@ -203,7 +204,7 @@ namespace basegfx
 
         // predefines for calls to this methods before method implementation
 
-        void findCuts(const B2DPolygon& rCandidate, temporaryPointVector& rTempPoints);
+        void findCuts(const B2DPolygon& rCandidate, temporaryPointVector& rTempPoints, size_t* pPointLimit = nullptr);
         void findTouches(const B2DPolygon& rEdgePolygon, const B2DPolygon& rPointPolygon, temporaryPointVector& rTempPoints);
         void findCuts(const B2DPolygon& rCandidateA, const B2DPolygon& rCandidateB, temporaryPointVector& rTempPointsA, temporaryPointVector& rTempPointsB);
 
@@ -487,7 +488,7 @@ namespace basegfx
             }
         }
 
-        void findCuts(const B2DPolygon& rCandidate, temporaryPointVector& rTempPoints)
+        void findCuts(const B2DPolygon& rCandidate, temporaryPointVector& rTempPoints, size_t* pPointLimit)
         {
             // find out if there are edges with intersections (self-cuts). If yes, add
             // entries to rTempPoints accordingly
@@ -588,6 +589,9 @@ namespace basegfx
                             findEdgeCutsTwoEdges(aCurrA, aNextA, aCurrB, aNextB, a, b, rTempPoints, rTempPoints);
                         }
 
+                        if (pPointLimit && rTempPoints.size() > *pPointLimit)
+                            break;
+
                         // prepare next step
                         aCurrB = aNextB;
                     }
@@ -596,6 +600,14 @@ namespace basegfx
                     aCurrA = aNextA;
                 }
             }
+
+            if (pPointLimit)
+            {
+                if (rTempPoints.size() > *pPointLimit)
+                    *pPointLimit = 0;
+                else
+                    *pPointLimit -= rTempPoints.size();
+            }
         }
 
     } // end of anonymous namespace
@@ -841,14 +853,19 @@ namespace basegfx
 namespace basegfx::utils
 {
 
-        B2DPolygon addPointsAtCutsAndTouches(const B2DPolygon& rCandidate)
+        B2DPolygon addPointsAtCutsAndTouches(const B2DPolygon& rCandidate, size_t* pPointLimit)
         {
             if(rCandidate.count())
             {
                 temporaryPointVector aTempPoints;
 
                 findTouches(rCandidate, rCandidate, aTempPoints);
-                findCuts(rCandidate, aTempPoints);
+                findCuts(rCandidate, aTempPoints, pPointLimit);
+                if (pPointLimit && !*pPointLimit)
+                {
+                    SAL_WARN("basegfx", "addPointsAtCutsAndTouches hit point limit");
+                    return rCandidate;
+                }
 
                 return mergeTemporaryPointsAndPolygon(rCandidate, aTempPoints);
             }
@@ -858,7 +875,7 @@ namespace basegfx::utils
             }
         }
 
-        B2DPolyPolygon addPointsAtCutsAndTouches(const B2DPolyPolygon& rCandidate)
+        B2DPolyPolygon addPointsAtCutsAndTouches(const B2DPolyPolygon& rCandidate, size_t* pPointLimit)
         {
             const sal_uInt32 nCount(rCandidate.count());
 
@@ -880,7 +897,13 @@ namespace basegfx::utils
                     for(a = 0; a < nCount; a++)
                     {
                         // use polygons with solved self intersections
-                        pTempData[a].setPolygon(addPointsAtCutsAndTouches(rCandidate.getB2DPolygon(a)));
+                        pTempData[a].setPolygon(addPointsAtCutsAndTouches(rCandidate.getB2DPolygon(a), pPointLimit));
+                    }
+
+                    if (pPointLimit && !*pPointLimit)
+                    {
+                        SAL_WARN("basegfx", "addPointsAtCutsAndTouches hit point limit");
+                        return rCandidate;
                     }
 
                     // now cuts and touches between the polygons
diff --git a/basegfx/source/polygon/b2dpolypolygoncutter.cxx b/basegfx/source/polygon/b2dpolypolygoncutter.cxx
index ac1e10660607..ddec57374649 100644
--- a/basegfx/source/polygon/b2dpolypolygoncutter.cxx
+++ b/basegfx/source/polygon/b2dpolypolygoncutter.cxx
@@ -513,7 +513,7 @@ namespace basegfx
                 impSolve();
             }
 
-            explicit solver(const B2DPolyPolygon& rOriginal)
+            explicit solver(const B2DPolyPolygon& rOriginal, size_t* pPointLimit = nullptr)
             :   maOriginal(rOriginal),
                 mbIsCurve(false),
                 mbChanged(false)
@@ -523,7 +523,7 @@ namespace basegfx
                 if(!nOriginalCount)
                     return;
 
-                B2DPolyPolygon aGeometry(utils::addPointsAtCutsAndTouches(maOriginal));
+                B2DPolyPolygon aGeometry(utils::addPointsAtCutsAndTouches(maOriginal, pPointLimit));
                 aGeometry.removeDoublePoints();
                 aGeometry = utils::simplifyCurveSegments(aGeometry);
                 mbIsCurve = aGeometry.areControlPointsUsed();
@@ -684,11 +684,11 @@ namespace basegfx
 namespace basegfx::utils
 {
 
-        B2DPolyPolygon solveCrossovers(const B2DPolyPolygon& rCandidate)
+        B2DPolyPolygon solveCrossovers(const B2DPolyPolygon& rCandidate, size_t* pPointLimit)
         {
             if(rCandidate.count() > 0)
             {
-                solver aSolver(rCandidate);
+                solver aSolver(rCandidate, pPointLimit);
                 return aSolver.getB2DPolyPolygon();
             }
             else
diff --git a/include/basegfx/polygon/b2dpolygonclipper.hxx b/include/basegfx/polygon/b2dpolygonclipper.hxx
index bc7bd2f1913e..a6a2ce9d2fa0 100644
--- a/include/basegfx/polygon/b2dpolygonclipper.hxx
+++ b/include/basegfx/polygon/b2dpolygonclipper.hxx
@@ -51,7 +51,8 @@ namespace basegfx::utils
     // With filled polygons, You get all tools::PolyPolygon parts which were inside rClip.
     // The switch bInside decides if the parts inside the clip polygon or outside shall be created.
     // The clip polygon is always assumed closed, even when it's isClosed() is false.
-    BASEGFX_DLLPUBLIC B2DPolyPolygon clipPolyPolygonOnPolyPolygon(const B2DPolyPolygon& rCandidate, const B2DPolyPolygon& rClip, bool bInside, bool bStroke);
+    BASEGFX_DLLPUBLIC B2DPolyPolygon clipPolyPolygonOnPolyPolygon(const B2DPolyPolygon& rCandidate, const B2DPolyPolygon& rClip,
+                                                                  bool bInside, bool bStroke, size_t *pPointLimit = nullptr);
     BASEGFX_DLLPUBLIC B2DPolyPolygon clipPolygonOnPolyPolygon(const B2DPolygon& rCandidate, const B2DPolyPolygon& rClip, bool bInside, bool bStroke);
 
     // clip the given polygon against the given range. the resulting polygon will always contain
diff --git a/include/basegfx/polygon/b2dpolygoncutandtouch.hxx b/include/basegfx/polygon/b2dpolygoncutandtouch.hxx
index 55bea6f3ebcb..600ad1c5cd90 100644
--- a/include/basegfx/polygon/b2dpolygoncutandtouch.hxx
+++ b/include/basegfx/polygon/b2dpolygoncutandtouch.hxx
@@ -27,12 +27,12 @@ namespace basegfx::utils
 
 // look for self-intersections and self-touches (points on an edge) in given polygon and add
 // extra points there. Result will have no touches or intersections on an edge, only on points
-B2DPolygon addPointsAtCutsAndTouches(const B2DPolygon& rCandidate);
+B2DPolygon addPointsAtCutsAndTouches(const B2DPolygon& rCandidate, size_t* pPointLimit = nullptr);
 
 // look for polypolygon-intersections and polypolygon-touches (point of poly A on an edge of poly B) in given tools::PolyPolygon and add
 // extra points there. Result will have no touches or intersections between contained polygons on an edge, only on points. For
 // convenience, the correction for self-intersections for each member polygon will be used, too.
-B2DPolyPolygon addPointsAtCutsAndTouches(const B2DPolyPolygon& rCandidate);
+B2DPolyPolygon addPointsAtCutsAndTouches(const B2DPolyPolygon& rCandidate, size_t* pPointLimit = nullptr);
 
 // look for intersections of rCandidate with the edge from rStart to rEnd and add extra points there.
 // Points are only added in the range of the edge, not on the endless vector.
diff --git a/include/basegfx/polygon/b2dpolypolygoncutter.hxx b/include/basegfx/polygon/b2dpolypolygoncutter.hxx
index a5f4ba6626af..55dd29cf8e52 100644
--- a/include/basegfx/polygon/b2dpolypolygoncutter.hxx
+++ b/include/basegfx/polygon/b2dpolypolygoncutter.hxx
@@ -39,7 +39,8 @@ namespace basegfx::utils
             preparing step and to explicitly correct their
             orientations.
         */
-        BASEGFX_DLLPUBLIC B2DPolyPolygon solveCrossovers(const B2DPolyPolygon& rCandidate);
+        BASEGFX_DLLPUBLIC B2DPolyPolygon solveCrossovers(const B2DPolyPolygon& rCandidate,
+                                                         size_t* pPointLimit = nullptr);
 
         /** Solve all crossovers (aka self-intersections) in a Polygon
 
diff --git a/vcl/source/gdi/region.cxx b/vcl/source/gdi/region.cxx
index 5293aa52328b..1f947f9b68fc 100644
--- a/vcl/source/gdi/region.cxx
+++ b/vcl/source/gdi/region.cxx
@@ -989,12 +989,15 @@ void vcl::Region::Intersect( const vcl::Region& rRegion )
             return;
         }
 
+        static size_t gPointLimit = !utl::ConfigManager::IsFuzzing() ? SAL_MAX_SIZE : SAL_MAX_INT16;
+        size_t nPointLimit(gPointLimit);
         const basegfx::B2DPolyPolygon aClip(
             basegfx::utils::clipPolyPolygonOnPolyPolygon(
                 aOtherPolyPoly,
                 aThisPolyPoly,
                 true,
-                false));
+                false,
+                &nPointLimit));
         *this = vcl::Region( aClip );
         return;
     }