fix areaScaleFragmentShader.glsl with texture atlas (tdf#105277)

With a texture atlas the "texture" is just a subtexture of a larger texture,
so texture coordinates are not the full range between 0 and 1, but just
a part of it. Since areaScaleFragmentShader converts between pixel
and texture coordinates, the conversion needs to take this into account.

Change-Id: I9d29ffea52551d19ba681971a2b4f140a35b491c
Reviewed-on: https://gerrit.libreoffice.org/70774
Tested-by: Jenkins
Reviewed-by: Luboš Luňák <l.lunak@collabora.com>

3 files changed, 74 insertions, 40 deletions

diff --git a/vcl/opengl/gdiimpl.cxx b/vcl/opengl/gdiimpl.cxx
index 1c0d3ecc91e2..9d42d03149a5 100644
--- a/vcl/opengl/gdiimpl.cxx
+++ b/vcl/opengl/gdiimpl.cxx
@@ -1000,15 +1000,20 @@ bool scaleTexture(const rtl::Reference< OpenGLContext > &xContext,
     OpenGLTexture aScratchTex(nNewWidth, nNewHeight);
     OpenGLFramebuffer* pFramebuffer = xContext->AcquireFramebuffer(aScratchTex);
 
+    // From OpenGLSalBitmap::ImplScaleArea().
     pProgram->SetUniform1f("xscale", ixscale);
     pProgram->SetUniform1f("yscale", iyscale);
     pProgram->SetUniform1i("swidth", nWidth);
     pProgram->SetUniform1i("sheight", nHeight);
-    // For converting between <0,nWidth-1> and <0.0,1.0> coordinate systems.
-    pProgram->SetUniform1f("xsrcconvert", 1.0 / (nWidth - 1));
-    pProgram->SetUniform1f("ysrcconvert", 1.0 / (nHeight - 1));
-    pProgram->SetUniform1f("xdestconvert", 1.0 * (nNewWidth - 1));
-    pProgram->SetUniform1f("ydestconvert", 1.0 * (nNewHeight - 1));
+    // For converting between <0,nWidth> and <0.0,1.0> coordinate systems.
+    GLfloat srcCoords[ 8 ];
+    rTexture.GetWholeCoord( srcCoords );
+    pProgram->SetUniform1f( "xoffset", srcCoords[ 0 ] );
+    pProgram->SetUniform1f( "yoffset", srcCoords[ 1 ] );
+    pProgram->SetUniform1f( "xtopixelratio", nNewWidth / ( srcCoords[ 4 ] - srcCoords[ 0 ] ));
+    pProgram->SetUniform1f( "ytopixelratio", nNewHeight / ( srcCoords[ 5 ] - srcCoords[ 1 ] ));
+    pProgram->SetUniform1f( "xfrompixelratio", ( srcCoords[ 4 ] - srcCoords[ 0 ] ) / nWidth );
+    pProgram->SetUniform1f( "yfrompixelratio", ( srcCoords[ 5 ] - srcCoords[ 1 ] ) / nHeight );
 
     pProgram->SetTexture("sampler", rTexture);
     pProgram->DrawTexture(rTexture);
@@ -1154,8 +1159,10 @@ void OpenGLSalGraphicsImpl::DrawTransformedTexture(
         {
             mpProgram->SetUniform1i( "xscale", ixscale );
             mpProgram->SetUniform1i( "yscale", iyscale );
-            mpProgram->SetUniform1f( "xstep", 1.0 / nWidth );
-            mpProgram->SetUniform1f( "ystep", 1.0 / nHeight );
+            GLfloat srcCoords[ 8 ];
+            aInTexture.GetWholeCoord( srcCoords );
+            mpProgram->SetUniform1f( "xstep", ( srcCoords[ 4 ] - srcCoords[ 0 ] ) / nWidth );
+            mpProgram->SetUniform1f( "ystep", ( srcCoords[ 5 ] - srcCoords[ 1 ] ) / nHeight );
             mpProgram->SetUniform1f( "ratio", 1.0 / ( ixscale * iyscale ));
         }
         else if (nHeight > 1 && nWidth > 1)
@@ -1165,10 +1172,14 @@ void OpenGLSalGraphicsImpl::DrawTransformedTexture(
             mpProgram->SetUniform1i( "swidth", nWidth );
             mpProgram->SetUniform1i( "sheight", nHeight );
             // For converting between <0,nWidth-1> and <0.0,1.0> coordinate systems.
-            mpProgram->SetUniform1f( "xsrcconvert", 1.0 / ( nWidth - 1 ));
-            mpProgram->SetUniform1f( "ysrcconvert", 1.0 / ( nHeight - 1 ));
-            mpProgram->SetUniform1f( "xdestconvert", 1.0 * (( nWidth / ixscale ) - 1 ));
-            mpProgram->SetUniform1f( "ydestconvert", 1.0 * (( nHeight / iyscale ) - 1 ));
+            GLfloat srcCoords[ 8 ];
+            aInTexture.GetWholeCoord( srcCoords );
+            mpProgram->SetUniform1f( "xoffset", srcCoords[ 0 ] );
+            mpProgram->SetUniform1f( "yoffset", srcCoords[ 1 ] );
+            mpProgram->SetUniform1f( "xtopixelratio", ( nWidth / ixscale ) / ( srcCoords[ 4 ] - srcCoords[ 0 ] ));
+            mpProgram->SetUniform1f( "ytopixelratio", ( nHeight / iyscale ) / ( srcCoords[ 5 ] - srcCoords[ 1 ] ));
+            mpProgram->SetUniform1f( "xfrompixelratio", ( srcCoords[ 4 ] - srcCoords[ 0 ] ) / nWidth );
+            mpProgram->SetUniform1f( "yfrompixelratio", ( srcCoords[ 5 ] - srcCoords[ 1 ] ) / nHeight );
         }
     }
 
diff --git a/vcl/opengl/scale.cxx b/vcl/opengl/scale.cxx
index 2feaa183d25f..98f0f5ea76b8 100644
--- a/vcl/opengl/scale.cxx
+++ b/vcl/opengl/scale.cxx
@@ -254,8 +254,15 @@ bool OpenGLSalBitmap::ImplScaleArea( const rtl::Reference< OpenGLContext > &xCon
     {
         pProgram->SetUniform1i( "xscale", ixscale );
         pProgram->SetUniform1i( "yscale", iyscale );
-        pProgram->SetUniform1f( "xstep", 1.0 / mnWidth );
-        pProgram->SetUniform1f( "ystep", 1.0 / mnHeight );
+        // The shader operates on pixels in the surrounding area, so it's necessary
+        // to know the step in texture coordinates to get to the next pixel.
+        // With a texture atlas the "texture" is just a subtexture of a larger texture,
+        // so while with a normal texture we'd map between <0.0,1.0> and <0,mnWidth>,
+        // with a subtexture the texture coordinates range is smaller.
+        GLfloat srcCoords[ 8 ];
+        maTexture.GetWholeCoord( srcCoords );
+        pProgram->SetUniform1f( "xstep", ( srcCoords[ 4 ] - srcCoords[ 0 ] ) / mnWidth );
+        pProgram->SetUniform1f( "ystep", ( srcCoords[ 5 ] - srcCoords[ 1 ] ) / mnHeight );
         pProgram->SetUniform1f( "ratio", 1.0 / ( ixscale * iyscale ));
     }
     else
@@ -264,11 +271,21 @@ bool OpenGLSalBitmap::ImplScaleArea( const rtl::Reference< OpenGLContext > &xCon
         pProgram->SetUniform1f( "yscale", iyscale );
         pProgram->SetUniform1i( "swidth", mnWidth );
         pProgram->SetUniform1i( "sheight", mnHeight );
-        // For converting between <0,mnWidth-1> and <0.0,1.0> coordinate systems.
-        pProgram->SetUniform1f( "xsrcconvert", 1.0 / ( mnWidth - 1 ));
-        pProgram->SetUniform1f( "ysrcconvert", 1.0 / ( mnHeight - 1 ));
-        pProgram->SetUniform1f( "xdestconvert", 1.0 * ( nNewWidth - 1 ));
-        pProgram->SetUniform1f( "ydestconvert", 1.0 * ( nNewHeight - 1 ));
+        // The shader internally actually operates on pixel coordinates,
+        // so it needs to know how to convert to those from the texture coordinates.
+        // With a simple texture that would mean converting e.g. between
+        // <0,mnWidth-1> and <0.0,1.0> coordinates.
+        // However with a texture atlas the "texture" is just a subtexture
+        // of a larger texture, so the texture coordinates need offset and ratio
+        // conversion too.
+        GLfloat srcCoords[ 8 ];
+        maTexture.GetWholeCoord( srcCoords );
+        pProgram->SetUniform1f( "xoffset", srcCoords[ 0 ] );
+        pProgram->SetUniform1f( "yoffset", srcCoords[ 1 ] );
+        pProgram->SetUniform1f( "xtopixelratio", nNewWidth / ( srcCoords[ 4 ] - srcCoords[ 0 ] ));
+        pProgram->SetUniform1f( "ytopixelratio", nNewHeight / ( srcCoords[ 5 ] - srcCoords[ 1 ] ));
+        pProgram->SetUniform1f( "xfrompixelratio", ( srcCoords[ 4 ] - srcCoords[ 0 ] ) / mnWidth );
+        pProgram->SetUniform1f( "yfrompixelratio", ( srcCoords[ 5 ] - srcCoords[ 1 ] ) / mnHeight );
     }
 
     pProgram->SetTexture( "sampler", maTexture );
@@ -302,11 +319,15 @@ bool OpenGLSalBitmap::ImplScaleArea( const rtl::Reference< OpenGLContext > &xCon
         pProgram->SetUniform1f("yscale", iyscale);
         pProgram->SetUniform1i("swidth", mnWidth);
         pProgram->SetUniform1i("sheight", mnHeight);
-        // For converting between <0,mnWidth-1> and <0.0,1.0> coordinate systems.
-        pProgram->SetUniform1f("xsrcconvert", 1.0 / (mnWidth - 1));
-        pProgram->SetUniform1f("ysrcconvert", 1.0 / (mnHeight - 1));
-        pProgram->SetUniform1f("xdestconvert", 1.0 * (nNewWidth - 1));
-        pProgram->SetUniform1f("ydestconvert", 1.0 * (nNewHeight - 1));
+
+        GLfloat srcCoords[ 8 ];
+        aScratchTex.GetWholeCoord( srcCoords );
+        pProgram->SetUniform1f( "xoffset", srcCoords[ 0 ] );
+        pProgram->SetUniform1f( "yoffset", srcCoords[ 1 ] );
+        pProgram->SetUniform1f( "xtopixelratio", nNewWidth / ( srcCoords[ 4 ] - srcCoords[ 0 ] ));
+        pProgram->SetUniform1f( "ytopixelratio", nNewHeight / ( srcCoords[ 5 ] - srcCoords[ 1 ] ));
+        pProgram->SetUniform1f( "xfrompixelratio", ( srcCoords[ 4 ] - srcCoords[ 0 ] ) / mnWidth );
+        pProgram->SetUniform1f( "yfrompixelratio", ( srcCoords[ 5 ] - srcCoords[ 1 ] ) / mnHeight );
 
         pProgram->SetTexture("sampler", aScratchTex);
         pProgram->DrawTexture(aScratchTex);
diff --git a/vcl/opengl/shaders/areaScaleFragmentShader.glsl b/vcl/opengl/shaders/areaScaleFragmentShader.glsl
index 07945e3ebe44..5dab5ba0114d 100644
--- a/vcl/opengl/shaders/areaScaleFragmentShader.glsl
+++ b/vcl/opengl/shaders/areaScaleFragmentShader.glsl
@@ -14,10 +14,12 @@ uniform int swidth;
 uniform int sheight;
 uniform float xscale;
 uniform float yscale;
-uniform float xsrcconvert;
-uniform float ysrcconvert;
-uniform float xdestconvert;
-uniform float ydestconvert;
+uniform float xoffset;
+uniform float yoffset;
+uniform float xfrompixelratio;
+uniform float yfrompixelratio;
+uniform float xtopixelratio;
+uniform float ytopixelratio;
 
 varying vec2 tex_coord;
 
@@ -28,23 +30,23 @@ varying vec2 mask_coord;
 uniform sampler2D mask;
 #endif
 
+#ifdef USE_REDUCED_REGISTER_VARIANT
+
 vec4 getTexel(int x, int y)
 {
-    vec2 offset = vec2(x * xsrcconvert, y * ysrcconvert);
-    vec4 texel = texture2D(sampler, offset);
+    vec2 pos = vec2( x * xfrompixelratio + xoffset, y * yfrompixelratio + yoffset );
+    vec4 texel = texture2D(sampler, pos);
 #ifdef MASKED
-    texel.a = 1.0 - texture2D(mask, offset).r;
+    texel.a = 1.0 - texture2D(mask, pos - tex_coord.st + mask_coord.st).r;
 #endif
     return texel;
 }
 
-#ifdef USE_REDUCED_REGISTER_VARIANT
-
 void main(void)
 {
     // Convert to pixel coordinates again.
-    int dx = int(tex_coord.s * xdestconvert);
-    int dy = int(tex_coord.t * ydestconvert);
+    int dx = int(( tex_coord.s - xoffset ) * xtopixelratio );
+    int dy = int(( tex_coord.t - yoffset ) * ytopixelratio );
 
     // Compute the range of source pixels which will make up this destination pixel.
     float fsx1 = min(dx * xscale,   float(swidth - 1));
@@ -124,8 +126,8 @@ void main(void)
 void main(void)
 {
     // Convert to pixel coordinates again.
-    int dx = int( tex_coord.s * xdestconvert );
-    int dy = int( tex_coord.t * ydestconvert );
+    int dx = int(( tex_coord.s - xoffset ) * xtopixelratio );
+    int dy = int(( tex_coord.t - yoffset ) * ytopixelratio );
 
     // How much each column/row will contribute to the resulting pixel.
     // Note: These values are always the same for the same X (or Y),
@@ -218,13 +220,13 @@ void main(void)
         xpos = 0;
         for( int x = xstart; x <= xend; ++x, ++xpos )
         {
-            vec2 offset = vec2( x * xsrcconvert, y * ysrcconvert );
+            vec2 pos = vec2( x * xfrompixelratio + xoffset, y * yfrompixelratio + yoffset );
 #ifndef MASKED
-            tmp += texture2D( sampler, offset ) * xratio[ xpos ];
+            tmp += texture2D( sampler, pos ) * xratio[ xpos ];
 #else
             vec4 texel;
-            texel = texture2D( sampler, offset );
-            texel.a = 1.0 - texture2D( mask, offset ).r;
+            texel = texture2D( sampler, pos );
+            texel.a = 1.0 - texture2D( mask, pos - tex_coord.st + mask_coord.st ).r;
             tmp += texel * xratio[ xpos ];
 #endif
         }