sal: rtlRandomPool: require OS random device, abort if not present

Both rtl_random_createPool() and rtl_random_getBytes() first try to get
random data from the OS, via /dev/urandom or rand_s() (documented to
call RtlGenRandom(), see [1]).

In case this does not succeed, there is a fallback to a custom
implementation of a PRNG of unknown design that has never been
substantially changed since initial CVS import, and is presumably not
what would be considered state of the art today, particularly if there's
no actual entropy available to seed it.

Except for a few miscellaneous usages in URE (presumably to avoid
dependencies on non-URE libs), rtlRandomPool is almost always used to
generate material for encryption of documents, which is demanding and
probably beyond what a pure user-space PRNG implementation without
entropy from the OS can provide.

So remove the custom PRNG and instead abort() if reading from the OS
random device fails for whatever reason.

rtl_random_addBytes() becomes a no-op and is therefore deprecated.

Presumably the only kind of environment where random device would be
unavailable in practice is running in some sort of chroot or container
that is missing the device or has incorrect permissions on it; better to
fail hard than to produce encrypted documents of questionable security.

[1] https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/rand-s?view=msvc-170

Change-Id: I3f020c2d11570f8351381d70188ce59bfec9f720
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/163056
Tested-by: Jenkins
Reviewed-by: Michael Stahl <michael.stahl@allotropia.de>

1 files changed, 6 insertions, 234 deletions

diff --git a/sal/rtl/random.cxx b/sal/rtl/random.cxx
index 418358b22e22..8420d4ca8049 100644
--- a/sal/rtl/random.cxx
+++ b/sal/rtl/random.cxx
@@ -19,246 +19,25 @@
 
 #include <sal/config.h>
 
-#include <cmath>
+#include <cstdio>
+#include <cstdlib>
 
 #include <sal/types.h>
-#include <o3tl/temporary.hxx>
-#include <osl/thread.h>
-#include <osl/thread.hxx>
-#include <osl/time.h>
 #include <rtl/alloc.h>
-#include <rtl/digest.h>
 #include <rtl/random.h>
 #include <oslrandom.h>
 
-#define RTL_RANDOM_RNG_1(a) ((a) * 16807L)
-#define RTL_RANDOM_RNG_2(a) ((a) * 65539L)
-
-#define RTL_RANDOM_RNG(x, y, z) \
-{ \
-    (x) = 170 * ((x) % 178) - 63 * ((x) / 178); \
-    if ((x) < 0) (x) += 30328; \
-    \
-    (y) = 171 * ((y) % 177) -  2 * ((y) / 177); \
-    if ((y) < 0) (y) += 30269; \
-    \
-    (z) = 172 * ((z) % 176) - 35 * ((z) / 176); \
-    if ((z) < 0) (z) += 30307; \
-}
-
-namespace {
-
-struct RandomData_Impl
-{
-    sal_Int16 m_nX;
-    sal_Int16 m_nY;
-    sal_Int16 m_nZ;
-};
-
-}
-
-static double data (RandomData_Impl *pImpl);
-
-#define RTL_RANDOM_DIGEST      rtl_Digest_AlgorithmMD5
-#define RTL_RANDOM_SIZE_DIGEST RTL_DIGEST_LENGTH_MD5
-#define RTL_RANDOM_SIZE_POOL   1023
-
 namespace {
 
 struct RandomPool_Impl
 {
-    rtlDigest  m_hDigest;
-    sal_uInt8  m_pDigest[RTL_RANDOM_SIZE_DIGEST];
-    sal_uInt8  m_pData[RTL_RANDOM_SIZE_POOL + 1];
-    sal_uInt32 m_nData;
-    sal_uInt32 m_nIndex;
-    sal_uInt32 m_nCount;
 };
 
 }
 
-static bool initPool(RandomPool_Impl *pImpl);
-
-static void seedPool(
-    RandomPool_Impl *pImpl, const sal_uInt8 *pBuffer, sal_Size nBufLen);
-
-static void readPool(
-    RandomPool_Impl *pImpl, sal_uInt8 *pBuffer, sal_Size nBufLen);
-
-static double data(RandomData_Impl *pImpl)
-{
-    double random;
-
-    RTL_RANDOM_RNG (pImpl->m_nX, pImpl->m_nY, pImpl->m_nZ);
-    random = ((static_cast<double>(pImpl->m_nX) / 30328.0) +
-              (static_cast<double>(pImpl->m_nY) / 30269.0) +
-              (static_cast<double>(pImpl->m_nZ) / 30307.0)   );
-
-    return std::modf(random, &o3tl::temporary(double()));
-}
-
-static bool initPool(RandomPool_Impl *pImpl)
-{
-    pImpl->m_hDigest = rtl_digest_create(RTL_RANDOM_DIGEST);
-    if (pImpl->m_hDigest)
-    {
-        oslThreadIdentifier tid;
-        TimeValue tv;
-        RandomData_Impl rd;
-        double seed;
-
-        /* The use of uninitialized stack variables as a way to
-         * enhance the entropy of the random pool triggers
-         * memory checkers like purify and valgrind.
-         */
-
-        /*
-        seedPool (pImpl, (sal_uInt8*)&tid, sizeof(tid));
-        seedPool (pImpl, (sal_uInt8*)&tv, sizeof(tv));
-        seedPool (pImpl, (sal_uInt8*)&rd, sizeof(rd));
-        */
-
-        tid = osl::Thread::getCurrentIdentifier();
-        tid = RTL_RANDOM_RNG_2(RTL_RANDOM_RNG_1(tid));
-        seedPool (pImpl, reinterpret_cast< sal_uInt8* >(&tid), sizeof(tid));
-
-        osl_getSystemTime (&tv);
-        tv.Seconds = RTL_RANDOM_RNG_2(tv.Seconds);
-        tv.Nanosec = RTL_RANDOM_RNG_2(tv.Nanosec);
-        seedPool (pImpl, reinterpret_cast< sal_uInt8* >(&tv), sizeof(tv));
-
-        rd.m_nX = static_cast<sal_Int16>(((tid        >> 1) << 1) + 1);
-        rd.m_nY = static_cast<sal_Int16>(((tv.Seconds >> 1) << 1) + 1);
-        rd.m_nZ = static_cast<sal_Int16>(((tv.Nanosec >> 1) << 1) + 1);
-        seedPool (pImpl, reinterpret_cast< sal_uInt8* >(&rd), sizeof(rd));
-
-        while (pImpl->m_nData < RTL_RANDOM_SIZE_POOL)
-        {
-            seed = data (&rd);
-            seedPool (pImpl, reinterpret_cast< sal_uInt8* >(&seed), sizeof(seed));
-        }
-        return true;
-    }
-    return false;
-}
-
-static void seedPool(
-    RandomPool_Impl *pImpl, const sal_uInt8 *pBuffer, sal_Size nBufLen)
-{
-    sal_Size i;
-    sal_sSize j, k;
-
-    for (i = 0; i < nBufLen; i += RTL_RANDOM_SIZE_DIGEST)
-    {
-        j = nBufLen - i;
-        if (j > RTL_RANDOM_SIZE_DIGEST)
-            j = RTL_RANDOM_SIZE_DIGEST;
-
-        rtl_digest_update(
-            pImpl->m_hDigest, pImpl->m_pDigest, RTL_RANDOM_SIZE_DIGEST);
-
-        k = (pImpl->m_nIndex + j) - RTL_RANDOM_SIZE_POOL;
-        if (k > 0)
-        {
-            rtl_digest_update(
-                pImpl->m_hDigest, &(pImpl->m_pData[pImpl->m_nIndex]), j - k);
-            rtl_digest_update(
-                pImpl->m_hDigest, &(pImpl->m_pData[0]), k);
-        }
-        else
-        {
-            rtl_digest_update(
-                pImpl->m_hDigest, &(pImpl->m_pData[pImpl->m_nIndex]), j);
-        }
-
-        rtl_digest_update(pImpl->m_hDigest, pBuffer, j);
-        pBuffer += j;
-
-        rtl_digest_get(
-            pImpl->m_hDigest, pImpl->m_pDigest, RTL_RANDOM_SIZE_DIGEST);
-        for (k = 0; k < j; k++)
-        {
-            pImpl->m_pData[pImpl->m_nIndex++] ^= pImpl->m_pDigest[k];
-            if (pImpl->m_nIndex >= RTL_RANDOM_SIZE_POOL)
-            {
-                pImpl->m_nData  = RTL_RANDOM_SIZE_POOL;
-                pImpl->m_nIndex = 0;
-            }
-        }
-    }
-
-    if (pImpl->m_nIndex > pImpl->m_nData)
-        pImpl->m_nData = pImpl->m_nIndex;
-}
-
-static void readPool (
-    RandomPool_Impl *pImpl, sal_uInt8 *pBuffer, sal_Size nBufLen)
-{
-    sal_Int32 j, k;
-
-    while (nBufLen > 0)
-    {
-        j = nBufLen;
-        if (j > RTL_RANDOM_SIZE_DIGEST/2)
-            j = RTL_RANDOM_SIZE_DIGEST/2;
-        nBufLen -= j;
-
-        rtl_digest_update(
-            pImpl->m_hDigest,
-            &(pImpl->m_pDigest[RTL_RANDOM_SIZE_DIGEST/2]),
-            RTL_RANDOM_SIZE_DIGEST/2);
-
-        k = (pImpl->m_nIndex + j) - pImpl->m_nData;
-        if (k > 0)
-        {
-            rtl_digest_update(
-                pImpl->m_hDigest, &(pImpl->m_pData[pImpl->m_nIndex]), j - k);
-            rtl_digest_update(
-                pImpl->m_hDigest, &(pImpl->m_pData[0]), k);
-        }
-        else
-        {
-            rtl_digest_update(
-                pImpl->m_hDigest, &(pImpl->m_pData[pImpl->m_nIndex]), j);
-        }
-
-        rtl_digest_get(
-            pImpl->m_hDigest, pImpl->m_pDigest, RTL_RANDOM_SIZE_DIGEST);
-        for (k = 0; k < j; k++)
-        {
-            if (pImpl->m_nIndex >= pImpl->m_nData)
-                pImpl->m_nIndex = 0;
-
-            pImpl->m_pData[pImpl->m_nIndex++] ^= pImpl->m_pDigest[k];
-            *pBuffer++ = pImpl->m_pDigest[k + RTL_RANDOM_SIZE_DIGEST/2];
-        }
-    }
-
-    pImpl->m_nCount++;
-    rtl_digest_update(
-        pImpl->m_hDigest, &(pImpl->m_nCount), sizeof(pImpl->m_nCount));
-    rtl_digest_update(
-        pImpl->m_hDigest, pImpl->m_pDigest, RTL_RANDOM_SIZE_DIGEST);
-    rtl_digest_get(
-        pImpl->m_hDigest, pImpl->m_pDigest, RTL_RANDOM_SIZE_DIGEST);
-}
-
 rtlRandomPool SAL_CALL rtl_random_createPool() SAL_THROW_EXTERN_C()
 {
-    /* try to get system random number, if it fail fall back on own pool */
     RandomPool_Impl *pImpl = static_cast< RandomPool_Impl* >(rtl_allocateZeroMemory(sizeof(RandomPool_Impl)));
-    if (pImpl)
-    {
-        char sanity[4];
-        if (!osl_get_system_random_data(sanity, 4))
-        {
-            if (!initPool(pImpl))
-            {
-                rtl_freeZeroMemory(pImpl, sizeof(RandomPool_Impl));
-                pImpl = nullptr;
-            }
-        }
-    }
     return static_cast< rtlRandomPool >(pImpl);
 }
 
@@ -267,15 +46,12 @@ void SAL_CALL rtl_random_destroyPool(rtlRandomPool Pool) SAL_THROW_EXTERN_C()
     RandomPool_Impl *pImpl = static_cast< RandomPool_Impl* >(Pool);
     if (pImpl)
     {
-        if (pImpl->m_hDigest)
-            rtl_digest_destroy(pImpl->m_hDigest);
-
         rtl_freeZeroMemory (pImpl, sizeof(RandomPool_Impl));
     }
 }
 
 rtlRandomError SAL_CALL rtl_random_addBytes(
-    rtlRandomPool Pool, const void *Buffer, sal_Size Bytes) SAL_THROW_EXTERN_C()
+    rtlRandomPool Pool, const void *Buffer, sal_Size /*Bytes*/) SAL_THROW_EXTERN_C()
 {
     RandomPool_Impl *pImpl = static_cast< RandomPool_Impl* >(Pool);
     const sal_uInt8 *pBuffer = static_cast< const sal_uInt8* >(Buffer);
@@ -283,9 +59,6 @@ rtlRandomError SAL_CALL rtl_random_addBytes(
     if (!pImpl || !pBuffer)
         return rtl_Random_E_Argument;
 
-    if (pImpl->m_hDigest)
-        seedPool (pImpl, pBuffer, Bytes);
-
     return rtl_Random_E_None;
 }
 
@@ -298,11 +71,10 @@ rtlRandomError SAL_CALL rtl_random_getBytes (
     if (!pImpl || !pBuffer)
         return rtl_Random_E_Argument;
 
-    if (pImpl->m_hDigest || !osl_get_system_random_data(static_cast< char* >(Buffer), Bytes))
+    if (!osl_get_system_random_data(static_cast<char*>(Buffer), Bytes))
     {
-        if (!pImpl->m_hDigest && !initPool(pImpl))
-            return rtl_Random_E_Unknown;
-        readPool(pImpl, pBuffer, Bytes);
+        ::std::fprintf(stderr, "rtl_random_getBytes(): cannot read random device, aborting.\n");
+        ::std::abort();
     }
     return rtl_Random_E_None;
 }