Mercurial > dropbear
view libtomcrypt/src/pk/rsa/rsa_sign_hash.c @ 1423:c1c3d5943bfc
Fix null pointer dereference found by libfuzzer
author | Matt Johnston <matt@ucc.asn.au> |
---|---|
date | Sun, 21 May 2017 18:53:09 +0800 |
parents | 0cbe8f6dbf9e |
children | f849a5ca2efc |
line wrap: on
line source
/* LibTomCrypt, modular cryptographic library -- Tom St Denis * * LibTomCrypt is a library that provides various cryptographic * algorithms in a highly modular and flexible manner. * * The library is free for all purposes without any express * guarantee it works. * * Tom St Denis, [email protected], http://libtomcrypt.com */ #include "tomcrypt.h" /** @file rsa_sign_hash.c RSA PKCS #1 v1.5 and v2 PSS sign hash, Tom St Denis and Andreas Lange */ #ifdef MRSA /** PKCS #1 pad then sign @param in The hash to sign @param inlen The length of the hash to sign (octets) @param out [out] The signature @param outlen [in/out] The max size and resulting size of the signature @param padding Type of padding (LTC_PKCS_1_PSS or LTC_PKCS_1_V1_5) @param prng An active PRNG state @param prng_idx The index of the PRNG desired @param hash_idx The index of the hash desired @param saltlen The length of the salt desired (octets) @param key The private RSA key to use @return CRYPT_OK if successful */ int rsa_sign_hash_ex(const unsigned char *in, unsigned long inlen, unsigned char *out, unsigned long *outlen, int padding, prng_state *prng, int prng_idx, int hash_idx, unsigned long saltlen, rsa_key *key) { unsigned long modulus_bitlen, modulus_bytelen, x, y; int err; LTC_ARGCHK(in != NULL); LTC_ARGCHK(out != NULL); LTC_ARGCHK(outlen != NULL); LTC_ARGCHK(key != NULL); /* valid padding? */ if ((padding != LTC_PKCS_1_V1_5) && (padding != LTC_PKCS_1_PSS)) { return CRYPT_PK_INVALID_PADDING; } if (padding == LTC_PKCS_1_PSS) { /* valid prng and hash ? */ if ((err = prng_is_valid(prng_idx)) != CRYPT_OK) { return err; } if ((err = hash_is_valid(hash_idx)) != CRYPT_OK) { return err; } } /* get modulus len in bits */ modulus_bitlen = mp_count_bits((key->N)); /* outlen must be at least the size of the modulus */ modulus_bytelen = mp_unsigned_bin_size((key->N)); if (modulus_bytelen > *outlen) { *outlen = modulus_bytelen; return CRYPT_BUFFER_OVERFLOW; } if (padding == LTC_PKCS_1_PSS) { /* PSS pad the key */ x = *outlen; if ((err = pkcs_1_pss_encode(in, inlen, saltlen, prng, prng_idx, hash_idx, modulus_bitlen, out, &x)) != CRYPT_OK) { return err; } } else { /* PKCS #1 v1.5 pad the hash */ unsigned char *tmpin; ltc_asn1_list digestinfo[2], siginfo[2]; /* not all hashes have OIDs... so sad */ if (hash_descriptor[hash_idx].OIDlen == 0) { return CRYPT_INVALID_ARG; } /* construct the SEQUENCE SEQUENCE { SEQUENCE {hashoid OID blah NULL } hash OCTET STRING } */ LTC_SET_ASN1(digestinfo, 0, LTC_ASN1_OBJECT_IDENTIFIER, hash_descriptor[hash_idx].OID, hash_descriptor[hash_idx].OIDlen); LTC_SET_ASN1(digestinfo, 1, LTC_ASN1_NULL, NULL, 0); LTC_SET_ASN1(siginfo, 0, LTC_ASN1_SEQUENCE, digestinfo, 2); LTC_SET_ASN1(siginfo, 1, LTC_ASN1_OCTET_STRING, in, inlen); /* allocate memory for the encoding */ y = mp_unsigned_bin_size(key->N); tmpin = XMALLOC(y); if (tmpin == NULL) { return CRYPT_MEM; } if ((err = der_encode_sequence(siginfo, 2, tmpin, &y)) != CRYPT_OK) { XFREE(tmpin); return err; } x = *outlen; if ((err = pkcs_1_v1_5_encode(tmpin, y, LTC_PKCS_1_EMSA, modulus_bitlen, NULL, 0, out, &x)) != CRYPT_OK) { XFREE(tmpin); return err; } XFREE(tmpin); } /* RSA encode it */ return ltc_mp.rsa_me(out, x, out, outlen, PK_PRIVATE, key); } #endif /* MRSA */ /* $Source: /cvs/libtom/libtomcrypt/src/pk/rsa/rsa_sign_hash.c,v $ */ /* $Revision: 1.9 $ */ /* $Date: 2006/11/09 23:15:39 $ */