Implement RSASSA-PKCS1-v1_5 as specified in RFC 8017.

Based on an OpenSSL commit by David Benjamin.

Alex Gaynor and Paul Kehrer from the pyca/cryptography Python library
reported that more than 200 "expected to fail" signatures among Project
Wycheproof's test vectors validated on LibreSSL. This patch makes them
all fail.

ok jsing

commit 608a026494c1e7a14f6d6cfcc5e4994fe2728836
Author: David Benjamin <[email protected]>
Date:   Sat Aug 20 13:35:17 2016 -0400

    Implement RSASSA-PKCS1-v1_5 as specified.

    RFC 3447, section 8.2.2, steps 3 and 4 states that verifiers must encode
    the DigestInfo struct and then compare the result against the public key
    operation result. This implies that one and only one encoding is legal.

    OpenSSL instead parses with crypto/asn1, then checks that the encoding
    round-trips, and allows some variations for the parameter. Sufficient
    laxness in this area can allow signature forgeries, as described in
    https://www.imperialviolet.org/2014/09/26/pkcs1.html

    Although there aren't known attacks against OpenSSL's current scheme,
    this change makes OpenSSL implement the algorithm as specified. This
    avoids the uncertainty and, more importantly, helps grow a healthy
    ecosystem. Laxness beyond the spec, particularly in implementations
    which enjoy wide use, risks harm to the ecosystem for all. A signature
    producer which only tests against OpenSSL may not notice bugs and
    accidentally become widely deployed. Thus implementations have a
    responsibility to honor the specification as tightly as is practical.

    In some cases, the damage is permanent and the spec deviation and
    security risk becomes a tax all implementors must forever pay, but not
    here. Both BoringSSL and Go successfully implemented and deployed
    RSASSA-PKCS1-v1_5 as specified since their respective beginnings, so
    this change should be compatible enough to pin down in future OpenSSL
    releases.

    See also https://tools.ietf.org/html/draft-thomson-postel-was-wrong-00

    As a bonus, by not having to deal with sign/verify differences, this
    version is also somewhat clearer. It also more consistently enforces
    digest lengths in the verify_recover codepath. The NID_md5_sha1 codepath
    wasn't quite doing this right.

    Reviewed-by: Kurt Roeckx <[email protected]>
    Reviewed-by: Rich Salz <[email protected]>

    GH: #1474

lib/libcrypto/rsa/rsa_sign.c

@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_sign.c,v 1.29 2017/05/02 03:59:45 deraadt Exp $ */
+/* $OpenBSD: rsa_sign.c,v 1.30 2018/07/23 17:37:17 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young ([email protected])
  * All rights reserved.
  *
@@ -70,168 +70,196 @@
 /* Size of an SSL signature: MD5+SHA1 */
 #define SSL_SIG_LENGTH 36
 
-int
-RSA_sign(int type, const unsigned char *m, unsigned int m_len,
- unsigned char *sigret, unsigned int *siglen, RSA *rsa)
+static int encode_pkcs1(unsigned char **, int *, int , const unsigned char *,
+ unsigned int);
+
+/*
+ * encode_pkcs1 encodes a DigestInfo prefix of hash `type' and digest `m', as
+ * described in EMSA-PKCS-v1_5-ENCODE, RFC 8017 section 9. step 2. This
+ * encodes the DigestInfo (T and tLen) but does not add the padding.
+ *
+ * On success, it returns one and sets `*out' to a newly allocated buffer
+ * containing the result and `*out_len' to its length. Freeing `*out' is
+ * the caller's responsibility. Failure is indicated by zero.
+ */
+static int
+encode_pkcs1(unsigned char **out, int *out_len, int type,
+ const unsigned char *m, unsigned int m_len)
 {
  X509_SIG sig;
- ASN1_TYPE parameter;
- int i, j, ret = 1;
- unsigned char *p, *tmps = NULL;
- const unsigned char *s = NULL;
  X509_ALGOR algor;
+ ASN1_TYPE parameter;
  ASN1_OCTET_STRING digest;
+ uint8_t *der = NULL;
+ int len;
+
+ sig.algor = &algor;
+ if ((sig.algor->algorithm = OBJ_nid2obj(type)) == NULL) {
+ RSAerror(RSA_R_UNKNOWN_ALGORITHM_TYPE);
+ return 0;
+ }
+ if (sig.algor->algorithm->length == 0) {
+ RSAerror(
+ RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
+ return 0;
+ }
+ parameter.type = V_ASN1_NULL;
+ parameter.value.ptr = NULL;
+ sig.algor->parameter = &parameter;
+
+ sig.digest = &digest;
+ sig.digest->data = (unsigned char*)m; /* TMP UGLY CAST */
+ sig.digest->length = m_len;
 
- if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_sign)
+ if ((len = i2d_X509_SIG(&sig, &der)) < 0)
+ return 0;
+
+ *out = der;
+ *out_len = len;
+
+ return 1;
+}
+
+int
+RSA_sign(int type, const unsigned char *m, unsigned int m_len,
+ unsigned char *sigret, unsigned int *siglen, RSA *rsa)
+{
+ const unsigned char *encoded = NULL;
+ unsigned char *tmps = NULL;
+ int encrypt_len, encoded_len = 0, ret = 0;
+
+ if ((rsa->flags & RSA_FLAG_SIGN_VER) && rsa->meth->rsa_sign != NULL)
  return rsa->meth->rsa_sign(type, m, m_len, sigret, siglen, rsa);
 
- /* Special case: SSL signature, just check the length */
+ /* Compute the encoded digest. */
  if (type == NID_md5_sha1) {
+ /*
+ * NID_md5_sha1 corresponds to the MD5/SHA1 combination in
+ * TLS 1.1 and earlier. It has no DigestInfo wrapper but
+ * otherwise is RSASSA-PKCS-v1.5.
+ */
  if (m_len != SSL_SIG_LENGTH) {
- RSAerror(RSA_R_INVALID_MESSAGE_LENGTH);
+ RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
  return 0;
  }
- i = SSL_SIG_LENGTH;
- s = m;
+ encoded_len = SSL_SIG_LENGTH;
+ encoded = m;
  } else {
- sig.algor = &algor;
- sig.algor->algorithm = OBJ_nid2obj(type);
- if (sig.algor->algorithm == NULL) {
- RSAerror(RSA_R_UNKNOWN_ALGORITHM_TYPE);
- return 0;
- }
- if (sig.algor->algorithm->length == 0) {
- RSAerror(RSA_R_THE_ASN1_OBJECT_IDENTIFIER_IS_NOT_KNOWN_FOR_THIS_MD);
- return 0;
- }
- parameter.type = V_ASN1_NULL;
- parameter.value.ptr = NULL;
- sig.algor->parameter = &parameter;
-
- sig.digest = &digest;
- sig.digest->data = (unsigned char *)m; /* TMP UGLY CAST */
- sig.digest->length = m_len;
-
- i = i2d_X509_SIG(&sig, NULL);
+ if (!encode_pkcs1(&tmps, &encoded_len, type, m, m_len))
+ goto err;
+ encoded = tmps;
  }
- j = RSA_size(rsa);
- if (i > j - RSA_PKCS1_PADDING_SIZE) {
+ if (encoded_len > RSA_size(rsa) - RSA_PKCS1_PADDING_SIZE) {
  RSAerror(RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY);
- return 0;
- }
- if (type != NID_md5_sha1) {
- tmps = malloc(j + 1);
- if (tmps == NULL) {
- RSAerror(ERR_R_MALLOC_FAILURE);
- return 0;
- }
- p = tmps;
- i2d_X509_SIG(&sig, &p);
- s = tmps;
+ goto err;
  }
- i = RSA_private_encrypt(i, s, sigret, rsa, RSA_PKCS1_PADDING);
- if (i <= 0)
- ret = 0;
- else
- *siglen = i;
-
- if (type != NID_md5_sha1)
- freezero(tmps, (unsigned int)j + 1);
+ if ((encrypt_len = RSA_private_encrypt(encoded_len, encoded, sigret,
+ rsa, RSA_PKCS1_PADDING)) <= 0)
+ goto err;
+
+ *siglen = encrypt_len;
+ ret = 1;
+
+ err:
+ freezero(tmps, (size_t)encoded_len);
  return (ret);
 }
 
+/*
+ * int_rsa_verify verifies an RSA signature in `sigbuf' using `rsa'. It may be
+ * called in two modes. If `rm' is NULL, it verifies the signature for the
+ * digest `m'. Otherwise, it recovers the digest from the signature, writing the
+ * digest to `rm' and the length to `*prm_len'. `type' is the NID of the digest
+ * algorithm to use. It returns one on successful verification and zero
+ * otherwise.
+ */
 int
-int_rsa_verify(int dtype, const unsigned char *m, unsigned int m_len,
+int_rsa_verify(int type, const unsigned char *m, unsigned int m_len,
  unsigned char *rm, size_t *prm_len, const unsigned char *sigbuf,
  size_t siglen, RSA *rsa)
 {
- int i, ret = 0, sigtype;
- unsigned char *s;
- X509_SIG *sig = NULL;
+ unsigned char *decrypt_buf, *encoded = NULL;
+ int decrypt_len, encoded_len = 0, ret = 0;
 
- if (siglen != (unsigned int)RSA_size(rsa)) {
+ if (siglen != (size_t)RSA_size(rsa)) {
  RSAerror(RSA_R_WRONG_SIGNATURE_LENGTH);
  return 0;
  }
 
- if ((dtype == NID_md5_sha1) && rm) {
- i = RSA_public_decrypt((int)siglen, sigbuf, rm, rsa,
- RSA_PKCS1_PADDING);
- if (i <= 0)
- return 0;
- *prm_len = i;
- return 1;
- }
-
- s = malloc(siglen);
- if (s == NULL) {
+ /* Recover the encoded digest. */
+ if ((decrypt_buf = malloc(siglen)) == NULL) {
  RSAerror(ERR_R_MALLOC_FAILURE);
  goto err;
  }
- if (dtype == NID_md5_sha1 && m_len != SSL_SIG_LENGTH) {
- RSAerror(RSA_R_INVALID_MESSAGE_LENGTH);
- goto err;
- }
- i = RSA_public_decrypt((int)siglen, sigbuf, s, rsa, RSA_PKCS1_PADDING);
-
- if (i <= 0)
+ if ((decrypt_len = RSA_public_decrypt((int)siglen, sigbuf, decrypt_buf,
+ rsa, RSA_PKCS1_PADDING)) <= 0)
  goto err;
+
+ if (type == NID_md5_sha1) {
+ /*
+ * NID_md5_sha1 corresponds to the MD5/SHA1 combination in
+ * TLS 1.1 and earlier. It has no DigestInfo wrapper but
+ * otherwise is RSASSA-PKCS1-v1_5.
+ */
+ if (decrypt_len != SSL_SIG_LENGTH) {
+ RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
+ goto err;
+ }
 
- /* Special case: SSL signature */
- if (dtype == NID_md5_sha1) {
- if (i != SSL_SIG_LENGTH || memcmp(s, m, SSL_SIG_LENGTH))
- RSAerror(RSA_R_BAD_SIGNATURE);
- else
- ret = 1;
+ if (rm != NULL) {
+ memcpy(rm, decrypt_buf, SSL_SIG_LENGTH);
+ *prm_len = SSL_SIG_LENGTH;
+ } else {
+ if (m_len != SSL_SIG_LENGTH) {
+ RSAerror(RSA_R_INVALID_MESSAGE_LENGTH);
+ goto err;
+ }
+ if (memcmp(decrypt_buf, m, SSL_SIG_LENGTH) != 0) {
+ RSAerror(RSA_R_BAD_SIGNATURE);
+ goto err;
+ }
+ }
  } else {
- const unsigned char *p = s;
+ /*
+ * If recovering the digest, extract a digest-sized output from
+ * the end of `decrypt_buf' for `encode_pkcs1', then compare the
+ * decryption output as in a standard verification.
+ */
+ if (rm != NULL) {
+ const EVP_MD *md;
 
- sig = d2i_X509_SIG(NULL, &p, (long)i);
+ if ((md = EVP_get_digestbynid(type)) == NULL) {
+ RSAerror(RSA_R_UNKNOWN_ALGORITHM_TYPE);
+ goto err;
+ }
+ if ((m_len = EVP_MD_size(md)) > (size_t)decrypt_len) {
+ RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
+ goto err;
+ }
+ m = decrypt_buf + decrypt_len - m_len;
+ }
 
- if (sig == NULL)
+ /* Construct the encoded digest and ensure it matches */
+ if (!encode_pkcs1(&encoded, &encoded_len, type, m, m_len))
  goto err;
 
- /* Excess data can be used to create forgeries */
- if (p != s + i) {
+ if (encoded_len != decrypt_len ||
+  memcmp(encoded, decrypt_buf, encoded_len) != 0) {
  RSAerror(RSA_R_BAD_SIGNATURE);
  goto err;
  }
 
- /* Parameters to the signature algorithm can also be used to
- create forgeries */
- if (sig->algor->parameter &&
- ASN1_TYPE_get(sig->algor->parameter) != V_ASN1_NULL) {
- RSAerror(RSA_R_BAD_SIGNATURE);
- goto err;
+ /* Output the recovered digest. */
+ if (rm != NULL) {
+ memcpy(rm, m, m_len);
+ *prm_len = m_len;
  }
-
- sigtype = OBJ_obj2nid(sig->algor->algorithm);
-
- if (sigtype != dtype) {
- RSAerror(RSA_R_ALGORITHM_MISMATCH);
- goto err;
- }
- if (rm) {
- const EVP_MD *md;
-
- md = EVP_get_digestbynid(dtype);
- if (md && (EVP_MD_size(md) != sig->digest->length))
- RSAerror(RSA_R_INVALID_DIGEST_LENGTH);
- else {
- memcpy(rm, sig->digest->data,
- sig->digest->length);
- *prm_len = sig->digest->length;
- ret = 1;
- }
- } else if ((unsigned int)sig->digest->length != m_len ||
- memcmp(m, sig->digest->data, m_len) != 0) {
- RSAerror(RSA_R_BAD_SIGNATURE);
- } else
- ret = 1;
  }
-err:
- X509_SIG_free(sig);
- freezero(s, (unsigned int)siglen);
+
+ ret = 1;
+ err:
+ freezero(encoded, (size_t)encoded_len);
+ freezero(decrypt_buf, siglen);
  return ret;
 }