view libtommath/bn_mp_prime_miller_rabin.c @ 1788:1fc0012b9c38

Fix handling of replies to global requests (#112) The current code assumes that all global requests want / need a reply. This isn't always true and the request itself indicates if it wants a reply or not. It causes a specific problem with [email protected] messages. These are sent by OpenSSH after authentication to inform the client of potential other host keys for the host. This can be used to add a new type of host key or to rotate host keys. The initial information message from the server is sent as a global request, but with want_reply set to false. This means that the server doesn't expect an answer to this message. Instead the client needs to send a prove request as a reply if it wants to receive proof of ownership for the host keys. The bug doesn't cause any current problems with due to how OpenSSH treats receiving the failure message. It instead treats it as a keepalive message and further ignores it. Arguably this is a protocol violation though of Dropbear and it is only accidental that it doesn't cause a problem with OpenSSH. The bug was found when adding host keys support to libssh, which is more strict protocol wise and treats the unexpected failure message an error, also see https://gitlab.com/libssh/libssh-mirror/-/merge_requests/145 for more information. The fix here is to honor the want_reply flag in the global request and to only send a reply if the other side expects a reply.
author Dirkjan Bussink <d.bussink@gmail.com>
date Thu, 10 Dec 2020 16:13:13 +0100
parents 1051e4eea25a
children
line wrap: on
line source

#include "tommath_private.h"
#ifdef BN_MP_PRIME_MILLER_RABIN_C
/* LibTomMath, multiple-precision integer library -- Tom St Denis */
/* SPDX-License-Identifier: Unlicense */

/* Miller-Rabin test of "a" to the base of "b" as described in
 * HAC pp. 139 Algorithm 4.24
 *
 * Sets result to 0 if definitely composite or 1 if probably prime.
 * Randomly the chance of error is no more than 1/4 and often
 * very much lower.
 */
mp_err mp_prime_miller_rabin(const mp_int *a, const mp_int *b, mp_bool *result)
{
   mp_int  n1, y, r;
   mp_err  err;
   int     s, j;

   /* default */
   *result = MP_NO;

   /* ensure b > 1 */
   if (mp_cmp_d(b, 1uL) != MP_GT) {
      return MP_VAL;
   }

   /* get n1 = a - 1 */
   if ((err = mp_init_copy(&n1, a)) != MP_OKAY) {
      return err;
   }
   if ((err = mp_sub_d(&n1, 1uL, &n1)) != MP_OKAY) {
      goto LBL_N1;
   }

   /* set 2**s * r = n1 */
   if ((err = mp_init_copy(&r, &n1)) != MP_OKAY) {
      goto LBL_N1;
   }

   /* count the number of least significant bits
    * which are zero
    */
   s = mp_cnt_lsb(&r);

   /* now divide n - 1 by 2**s */
   if ((err = mp_div_2d(&r, s, &r, NULL)) != MP_OKAY) {
      goto LBL_R;
   }

   /* compute y = b**r mod a */
   if ((err = mp_init(&y)) != MP_OKAY) {
      goto LBL_R;
   }
   if ((err = mp_exptmod(b, &r, a, &y)) != MP_OKAY) {
      goto LBL_Y;
   }

   /* if y != 1 and y != n1 do */
   if ((mp_cmp_d(&y, 1uL) != MP_EQ) && (mp_cmp(&y, &n1) != MP_EQ)) {
      j = 1;
      /* while j <= s-1 and y != n1 */
      while ((j <= (s - 1)) && (mp_cmp(&y, &n1) != MP_EQ)) {
         if ((err = mp_sqrmod(&y, a, &y)) != MP_OKAY) {
            goto LBL_Y;
         }

         /* if y == 1 then composite */
         if (mp_cmp_d(&y, 1uL) == MP_EQ) {
            goto LBL_Y;
         }

         ++j;
      }

      /* if y != n1 then composite */
      if (mp_cmp(&y, &n1) != MP_EQ) {
         goto LBL_Y;
      }
   }

   /* probably prime now */
   *result = MP_YES;
LBL_Y:
   mp_clear(&y);
LBL_R:
   mp_clear(&r);
LBL_N1:
   mp_clear(&n1);
   return err;
}
#endif