1 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
2  * project 2006.
3  */
4 /* ====================================================================
5  * Copyright (c) 2006 The OpenSSL Project.  All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  *
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in
16  *    the documentation and/or other materials provided with the
17  *    distribution.
18  *
19  * 3. All advertising materials mentioning features or use of this
20  *    software must display the following acknowledgment:
21  *    "This product includes software developed by the OpenSSL Project
22  *    for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
23  *
24  * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
25  *    endorse or promote products derived from this software without
26  *    prior written permission. For written permission, please contact
27  *    licensing@OpenSSL.org.
28  *
29  * 5. Products derived from this software may not be called "OpenSSL"
30  *    nor may "OpenSSL" appear in their names without prior written
31  *    permission of the OpenSSL Project.
32  *
33  * 6. Redistributions of any form whatsoever must retain the following
34  *    acknowledgment:
35  *    "This product includes software developed by the OpenSSL Project
36  *    for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
37  *
38  * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
39  * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
40  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
41  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE OpenSSL PROJECT OR
42  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
43  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
44  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
45  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
46  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
47  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
48  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
49  * OF THE POSSIBILITY OF SUCH DAMAGE.
50  * ====================================================================
51  *
52  * This product includes cryptographic software written by Eric Young
53  * (eay@cryptsoft.com).  This product includes software written by Tim
54  * Hudson (tjh@cryptsoft.com). */
55 
56 #include <openssl/evp.h>
57 
58 #include <openssl/asn1.h>
59 #include <openssl/asn1t.h>
60 #include <openssl/digest.h>
61 #include <openssl/err.h>
62 #include <openssl/mem.h>
63 #include <openssl/obj.h>
64 #include <openssl/rsa.h>
65 #include <openssl/x509.h>
66 
67 #include "../rsa/internal.h"
68 #include "internal.h"
69 
70 
rsa_pub_encode(X509_PUBKEY * pk,const EVP_PKEY * pkey)71 static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey) {
72   uint8_t *encoded = NULL;
73   int len;
74   len = i2d_RSAPublicKey(pkey->pkey.rsa, &encoded);
75 
76   if (len <= 0) {
77     return 0;
78   }
79 
80   if (!X509_PUBKEY_set0_param(pk, OBJ_nid2obj(EVP_PKEY_RSA), V_ASN1_NULL, NULL,
81                               encoded, len)) {
82     OPENSSL_free(encoded);
83     return 0;
84   }
85 
86   return 1;
87 }
88 
rsa_pub_decode(EVP_PKEY * pkey,X509_PUBKEY * pubkey)89 static int rsa_pub_decode(EVP_PKEY *pkey, X509_PUBKEY *pubkey) {
90   const uint8_t *p;
91   int pklen;
92   RSA *rsa;
93 
94   if (!X509_PUBKEY_get0_param(NULL, &p, &pklen, NULL, pubkey)) {
95     return 0;
96   }
97   rsa = d2i_RSAPublicKey(NULL, &p, pklen);
98   if (rsa == NULL) {
99     OPENSSL_PUT_ERROR(EVP, rsa_pub_decode, ERR_R_RSA_LIB);
100     return 0;
101   }
102   EVP_PKEY_assign_RSA(pkey, rsa);
103   return 1;
104 }
105 
rsa_pub_cmp(const EVP_PKEY * a,const EVP_PKEY * b)106 static int rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b) {
107   return BN_cmp(b->pkey.rsa->n, a->pkey.rsa->n) == 0 &&
108          BN_cmp(b->pkey.rsa->e, a->pkey.rsa->e) == 0;
109 }
110 
rsa_priv_encode(PKCS8_PRIV_KEY_INFO * p8,const EVP_PKEY * pkey)111 static int rsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey) {
112   uint8_t *rk = NULL;
113   int rklen;
114 
115   rklen = i2d_RSAPrivateKey(pkey->pkey.rsa, &rk);
116 
117   if (rklen <= 0) {
118     OPENSSL_PUT_ERROR(EVP, rsa_priv_encode, ERR_R_MALLOC_FAILURE);
119     return 0;
120   }
121 
122   /* TODO(fork): const correctness in next line. */
123   if (!PKCS8_pkey_set0(p8, (ASN1_OBJECT *)OBJ_nid2obj(NID_rsaEncryption), 0,
124                        V_ASN1_NULL, NULL, rk, rklen)) {
125     OPENSSL_PUT_ERROR(EVP, rsa_priv_encode, ERR_R_MALLOC_FAILURE);
126     return 0;
127   }
128 
129   return 1;
130 }
131 
rsa_priv_decode(EVP_PKEY * pkey,PKCS8_PRIV_KEY_INFO * p8)132 static int rsa_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8) {
133   const uint8_t *p;
134   int pklen;
135   RSA *rsa;
136 
137   if (!PKCS8_pkey_get0(NULL, &p, &pklen, NULL, p8)) {
138     OPENSSL_PUT_ERROR(EVP, rsa_priv_decode, ERR_R_MALLOC_FAILURE);
139     return 0;
140   }
141 
142   rsa = d2i_RSAPrivateKey(NULL, &p, pklen);
143   if (rsa == NULL) {
144     OPENSSL_PUT_ERROR(EVP, rsa_priv_decode, ERR_R_RSA_LIB);
145     return 0;
146   }
147 
148   EVP_PKEY_assign_RSA(pkey, rsa);
149   return 1;
150 }
151 
rsa_opaque(const EVP_PKEY * pkey)152 static int rsa_opaque(const EVP_PKEY *pkey) {
153   return RSA_is_opaque(pkey->pkey.rsa);
154 }
155 
rsa_supports_digest(const EVP_PKEY * pkey,const EVP_MD * md)156 static int rsa_supports_digest(const EVP_PKEY *pkey, const EVP_MD *md) {
157   return RSA_supports_digest(pkey->pkey.rsa, md);
158 }
159 
int_rsa_size(const EVP_PKEY * pkey)160 static int int_rsa_size(const EVP_PKEY *pkey) {
161   return RSA_size(pkey->pkey.rsa);
162 }
163 
rsa_bits(const EVP_PKEY * pkey)164 static int rsa_bits(const EVP_PKEY *pkey) {
165   return BN_num_bits(pkey->pkey.rsa->n);
166 }
167 
int_rsa_free(EVP_PKEY * pkey)168 static void int_rsa_free(EVP_PKEY *pkey) { RSA_free(pkey->pkey.rsa); }
169 
update_buflen(const BIGNUM * b,size_t * pbuflen)170 static void update_buflen(const BIGNUM *b, size_t *pbuflen) {
171   size_t i;
172 
173   if (!b) {
174     return;
175   }
176 
177   i = BN_num_bytes(b);
178   if (*pbuflen < i) {
179     *pbuflen = i;
180   }
181 }
182 
do_rsa_print(BIO * out,const RSA * rsa,int off,int include_private)183 static int do_rsa_print(BIO *out, const RSA *rsa, int off,
184                         int include_private) {
185   char *str;
186   const char *s;
187   uint8_t *m = NULL;
188   int ret = 0, mod_len = 0;
189   size_t buf_len = 0;
190 
191   update_buflen(rsa->n, &buf_len);
192   update_buflen(rsa->e, &buf_len);
193 
194   if (include_private) {
195     update_buflen(rsa->d, &buf_len);
196     update_buflen(rsa->p, &buf_len);
197     update_buflen(rsa->q, &buf_len);
198     update_buflen(rsa->dmp1, &buf_len);
199     update_buflen(rsa->dmq1, &buf_len);
200     update_buflen(rsa->iqmp, &buf_len);
201   }
202 
203   m = (uint8_t *)OPENSSL_malloc(buf_len + 10);
204   if (m == NULL) {
205     OPENSSL_PUT_ERROR(EVP, do_rsa_print, ERR_R_MALLOC_FAILURE);
206     goto err;
207   }
208 
209   if (rsa->n != NULL) {
210     mod_len = BN_num_bits(rsa->n);
211   }
212 
213   if (!BIO_indent(out, off, 128)) {
214     goto err;
215   }
216 
217   if (include_private && rsa->d) {
218     if (BIO_printf(out, "Private-Key: (%d bit)\n", mod_len) <= 0) {
219       goto err;
220     }
221     str = "modulus:";
222     s = "publicExponent:";
223   } else {
224     if (BIO_printf(out, "Public-Key: (%d bit)\n", mod_len) <= 0) {
225       goto err;
226     }
227     str = "Modulus:";
228     s = "Exponent:";
229   }
230   if (!ASN1_bn_print(out, str, rsa->n, m, off) ||
231       !ASN1_bn_print(out, s, rsa->e, m, off)) {
232     goto err;
233   }
234 
235   if (include_private) {
236     if (!ASN1_bn_print(out, "privateExponent:", rsa->d, m, off) ||
237         !ASN1_bn_print(out, "prime1:", rsa->p, m, off) ||
238         !ASN1_bn_print(out, "prime2:", rsa->q, m, off) ||
239         !ASN1_bn_print(out, "exponent1:", rsa->dmp1, m, off) ||
240         !ASN1_bn_print(out, "exponent2:", rsa->dmq1, m, off) ||
241         !ASN1_bn_print(out, "coefficient:", rsa->iqmp, m, off)) {
242       goto err;
243     }
244   }
245   ret = 1;
246 
247 err:
248   OPENSSL_free(m);
249   return ret;
250 }
251 
rsa_pub_print(BIO * bp,const EVP_PKEY * pkey,int indent,ASN1_PCTX * ctx)252 static int rsa_pub_print(BIO *bp, const EVP_PKEY *pkey, int indent,
253                          ASN1_PCTX *ctx) {
254   return do_rsa_print(bp, pkey->pkey.rsa, indent, 0);
255 }
256 
257 
rsa_priv_print(BIO * bp,const EVP_PKEY * pkey,int indent,ASN1_PCTX * ctx)258 static int rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent,
259                           ASN1_PCTX *ctx) {
260   return do_rsa_print(bp, pkey->pkey.rsa, indent, 1);
261 }
262 
263 /* Given an MGF1 Algorithm ID decode to an Algorithm Identifier */
rsa_mgf1_decode(X509_ALGOR * alg)264 static X509_ALGOR *rsa_mgf1_decode(X509_ALGOR *alg) {
265   const uint8_t *p;
266   int plen;
267 
268   if (alg == NULL ||
269       OBJ_obj2nid(alg->algorithm) != NID_mgf1 ||
270       alg->parameter->type != V_ASN1_SEQUENCE) {
271     return NULL;
272   }
273 
274   p = alg->parameter->value.sequence->data;
275   plen = alg->parameter->value.sequence->length;
276   return d2i_X509_ALGOR(NULL, &p, plen);
277 }
278 
rsa_pss_decode(const X509_ALGOR * alg,X509_ALGOR ** pmaskHash)279 static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg,
280                                       X509_ALGOR **pmaskHash) {
281   const uint8_t *p;
282   int plen;
283   RSA_PSS_PARAMS *pss;
284 
285   *pmaskHash = NULL;
286 
287   if (!alg->parameter || alg->parameter->type != V_ASN1_SEQUENCE) {
288     return NULL;
289   }
290   p = alg->parameter->value.sequence->data;
291   plen = alg->parameter->value.sequence->length;
292   pss = d2i_RSA_PSS_PARAMS(NULL, &p, plen);
293 
294   if (!pss) {
295     return NULL;
296   }
297 
298   *pmaskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
299 
300   return pss;
301 }
302 
rsa_pss_param_print(BIO * bp,RSA_PSS_PARAMS * pss,X509_ALGOR * maskHash,int indent)303 static int rsa_pss_param_print(BIO *bp, RSA_PSS_PARAMS *pss,
304                                X509_ALGOR *maskHash, int indent) {
305   int rv = 0;
306 
307   if (!pss) {
308     if (BIO_puts(bp, " (INVALID PSS PARAMETERS)\n") <= 0) {
309       return 0;
310     }
311     return 1;
312   }
313 
314   if (BIO_puts(bp, "\n") <= 0 ||
315       !BIO_indent(bp, indent, 128) ||
316       BIO_puts(bp, "Hash Algorithm: ") <= 0) {
317     goto err;
318   }
319 
320   if (pss->hashAlgorithm) {
321     if (i2a_ASN1_OBJECT(bp, pss->hashAlgorithm->algorithm) <= 0) {
322       goto err;
323     }
324   } else if (BIO_puts(bp, "sha1 (default)") <= 0) {
325     goto err;
326   }
327 
328   if (BIO_puts(bp, "\n") <= 0 ||
329       !BIO_indent(bp, indent, 128) ||
330       BIO_puts(bp, "Mask Algorithm: ") <= 0) {
331     goto err;
332   }
333 
334   if (pss->maskGenAlgorithm) {
335     if (i2a_ASN1_OBJECT(bp, pss->maskGenAlgorithm->algorithm) <= 0 ||
336         BIO_puts(bp, " with ") <= 0) {
337       goto err;
338     }
339 
340     if (maskHash) {
341       if (i2a_ASN1_OBJECT(bp, maskHash->algorithm) <= 0) {
342         goto err;
343       }
344     } else if (BIO_puts(bp, "INVALID") <= 0) {
345       goto err;
346     }
347   } else if (BIO_puts(bp, "mgf1 with sha1 (default)") <= 0) {
348     goto err;
349   }
350   BIO_puts(bp, "\n");
351 
352   if (!BIO_indent(bp, indent, 128) ||
353       BIO_puts(bp, "Salt Length: 0x") <= 0) {
354     goto err;
355   }
356 
357   if (pss->saltLength) {
358     if (i2a_ASN1_INTEGER(bp, pss->saltLength) <= 0) {
359       goto err;
360     }
361   } else if (BIO_puts(bp, "14 (default)") <= 0) {
362     goto err;
363   }
364   BIO_puts(bp, "\n");
365 
366   if (!BIO_indent(bp, indent, 128) ||
367       BIO_puts(bp, "Trailer Field: 0x") <= 0) {
368     goto err;
369   }
370 
371   if (pss->trailerField) {
372     if (i2a_ASN1_INTEGER(bp, pss->trailerField) <= 0) {
373       goto err;
374     }
375   } else if (BIO_puts(bp, "BC (default)") <= 0) {
376     goto err;
377   }
378   BIO_puts(bp, "\n");
379 
380   rv = 1;
381 
382 err:
383   return rv;
384 }
385 
rsa_sig_print(BIO * bp,const X509_ALGOR * sigalg,const ASN1_STRING * sig,int indent,ASN1_PCTX * pctx)386 static int rsa_sig_print(BIO *bp, const X509_ALGOR *sigalg,
387                          const ASN1_STRING *sig, int indent, ASN1_PCTX *pctx) {
388   if (OBJ_obj2nid(sigalg->algorithm) == NID_rsassaPss) {
389     int rv;
390     RSA_PSS_PARAMS *pss;
391     X509_ALGOR *maskHash;
392 
393     pss = rsa_pss_decode(sigalg, &maskHash);
394     rv = rsa_pss_param_print(bp, pss, maskHash, indent);
395     RSA_PSS_PARAMS_free(pss);
396     X509_ALGOR_free(maskHash);
397     if (!rv) {
398       return 0;
399     }
400   } else if (!sig && BIO_puts(bp, "\n") <= 0) {
401     return 0;
402   }
403 
404   if (sig) {
405     return X509_signature_dump(bp, sig, indent);
406   }
407   return 1;
408 }
409 
old_rsa_priv_decode(EVP_PKEY * pkey,const unsigned char ** pder,int derlen)410 static int old_rsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder,
411                                int derlen) {
412   RSA *rsa = d2i_RSAPrivateKey(NULL, pder, derlen);
413   if (rsa == NULL) {
414     OPENSSL_PUT_ERROR(EVP, old_rsa_priv_decode, ERR_R_RSA_LIB);
415     return 0;
416   }
417   EVP_PKEY_assign_RSA(pkey, rsa);
418   return 1;
419 }
420 
old_rsa_priv_encode(const EVP_PKEY * pkey,unsigned char ** pder)421 static int old_rsa_priv_encode(const EVP_PKEY *pkey, unsigned char **pder) {
422   return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
423 }
424 
425 /* allocate and set algorithm ID from EVP_MD, default SHA1 */
rsa_md_to_algor(X509_ALGOR ** palg,const EVP_MD * md)426 static int rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md) {
427   if (EVP_MD_type(md) == NID_sha1) {
428     return 1;
429   }
430   *palg = X509_ALGOR_new();
431   if (!*palg) {
432     return 0;
433   }
434   X509_ALGOR_set_md(*palg, md);
435   return 1;
436 }
437 
438 /* Allocate and set MGF1 algorithm ID from EVP_MD */
rsa_md_to_mgf1(X509_ALGOR ** palg,const EVP_MD * mgf1md)439 static int rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md) {
440   X509_ALGOR *algtmp = NULL;
441   ASN1_STRING *stmp = NULL;
442   *palg = NULL;
443 
444   if (EVP_MD_type(mgf1md) == NID_sha1) {
445     return 1;
446   }
447   /* need to embed algorithm ID inside another */
448   if (!rsa_md_to_algor(&algtmp, mgf1md) ||
449       !ASN1_item_pack(algtmp, ASN1_ITEM_rptr(X509_ALGOR), &stmp)) {
450     goto err;
451   }
452   *palg = X509_ALGOR_new();
453   if (!*palg) {
454     goto err;
455   }
456   X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp);
457   stmp = NULL;
458 
459 err:
460   ASN1_STRING_free(stmp);
461   X509_ALGOR_free(algtmp);
462   if (*palg) {
463     return 1;
464   }
465 
466   return 0;
467 }
468 
469 /* convert algorithm ID to EVP_MD, default SHA1 */
rsa_algor_to_md(X509_ALGOR * alg)470 static const EVP_MD *rsa_algor_to_md(X509_ALGOR *alg) {
471   const EVP_MD *md;
472   if (!alg) {
473     return EVP_sha1();
474   }
475   md = EVP_get_digestbyobj(alg->algorithm);
476   if (md == NULL) {
477     OPENSSL_PUT_ERROR(EVP, rsa_algor_to_md, EVP_R_UNKNOWN_DIGEST);
478   }
479   return md;
480 }
481 
482 /* convert MGF1 algorithm ID to EVP_MD, default SHA1 */
rsa_mgf1_to_md(X509_ALGOR * alg,X509_ALGOR * maskHash)483 static const EVP_MD *rsa_mgf1_to_md(X509_ALGOR *alg, X509_ALGOR *maskHash) {
484   const EVP_MD *md;
485   if (!alg) {
486     return EVP_sha1();
487   }
488   /* Check mask and lookup mask hash algorithm */
489   if (OBJ_obj2nid(alg->algorithm) != NID_mgf1) {
490     OPENSSL_PUT_ERROR(EVP, rsa_mgf1_to_md, EVP_R_UNSUPPORTED_MASK_ALGORITHM);
491     return NULL;
492   }
493   if (!maskHash) {
494     OPENSSL_PUT_ERROR(EVP, rsa_mgf1_to_md, EVP_R_UNSUPPORTED_MASK_PARAMETER);
495     return NULL;
496   }
497   md = EVP_get_digestbyobj(maskHash->algorithm);
498   if (md == NULL) {
499     OPENSSL_PUT_ERROR(EVP, rsa_mgf1_to_md, EVP_R_UNKNOWN_MASK_DIGEST);
500     return NULL;
501   }
502   return md;
503 }
504 
505 /* rsa_ctx_to_pss converts EVP_PKEY_CTX in PSS mode into corresponding
506  * algorithm parameter, suitable for setting as an AlgorithmIdentifier. */
rsa_ctx_to_pss(EVP_PKEY_CTX * pkctx)507 static ASN1_STRING *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx) {
508   const EVP_MD *sigmd, *mgf1md;
509   RSA_PSS_PARAMS *pss = NULL;
510   ASN1_STRING *os = NULL;
511   EVP_PKEY *pk = EVP_PKEY_CTX_get0_pkey(pkctx);
512   int saltlen, rv = 0;
513 
514   if (!EVP_PKEY_CTX_get_signature_md(pkctx, &sigmd) ||
515       !EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) ||
516       !EVP_PKEY_CTX_get_rsa_pss_saltlen(pkctx, &saltlen)) {
517     goto err;
518   }
519 
520   if (saltlen == -1) {
521     saltlen = EVP_MD_size(sigmd);
522   } else if (saltlen == -2) {
523     saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
524     if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0) {
525       saltlen--;
526     }
527   } else {
528     goto err;
529   }
530 
531   pss = RSA_PSS_PARAMS_new();
532   if (!pss) {
533     goto err;
534   }
535 
536   if (saltlen != 20) {
537     pss->saltLength = ASN1_INTEGER_new();
538     if (!pss->saltLength ||
539         !ASN1_INTEGER_set(pss->saltLength, saltlen)) {
540       goto err;
541     }
542   }
543 
544   if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd) ||
545       !rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md)) {
546     goto err;
547   }
548 
549   /* Finally create string with pss parameter encoding. */
550   if (!ASN1_item_pack(pss, ASN1_ITEM_rptr(RSA_PSS_PARAMS), &os)) {
551     goto err;
552   }
553   rv = 1;
554 
555 err:
556   if (pss) {
557     RSA_PSS_PARAMS_free(pss);
558   }
559   if (rv) {
560     return os;
561   }
562   if (os) {
563     ASN1_STRING_free(os);
564   }
565   return NULL;
566 }
567 
568 /* From PSS AlgorithmIdentifier set public key parameters. */
rsa_pss_to_ctx(EVP_MD_CTX * ctx,X509_ALGOR * sigalg,EVP_PKEY * pkey)569 static int rsa_pss_to_ctx(EVP_MD_CTX *ctx, X509_ALGOR *sigalg, EVP_PKEY *pkey) {
570   int ret = 0;
571   int saltlen;
572   const EVP_MD *mgf1md = NULL, *md = NULL;
573   RSA_PSS_PARAMS *pss;
574   X509_ALGOR *maskHash;
575   EVP_PKEY_CTX *pkctx;
576 
577   /* Sanity check: make sure it is PSS */
578   if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) {
579     OPENSSL_PUT_ERROR(EVP, rsa_pss_to_ctx, EVP_R_UNSUPPORTED_SIGNATURE_TYPE);
580     return 0;
581   }
582   /* Decode PSS parameters */
583   pss = rsa_pss_decode(sigalg, &maskHash);
584   if (pss == NULL) {
585     OPENSSL_PUT_ERROR(EVP, rsa_pss_to_ctx, EVP_R_INVALID_PSS_PARAMETERS);
586     goto err;
587   }
588 
589   mgf1md = rsa_mgf1_to_md(pss->maskGenAlgorithm, maskHash);
590   if (!mgf1md) {
591     goto err;
592   }
593   md = rsa_algor_to_md(pss->hashAlgorithm);
594   if (!md) {
595     goto err;
596   }
597 
598   saltlen = 20;
599   if (pss->saltLength) {
600     saltlen = ASN1_INTEGER_get(pss->saltLength);
601 
602     /* Could perform more salt length sanity checks but the main
603      * RSA routines will trap other invalid values anyway. */
604     if (saltlen < 0) {
605       OPENSSL_PUT_ERROR(EVP, rsa_pss_to_ctx, EVP_R_INVALID_SALT_LENGTH);
606       goto err;
607     }
608   }
609 
610   /* low-level routines support only trailer field 0xbc (value 1)
611    * and PKCS#1 says we should reject any other value anyway. */
612   if (pss->trailerField && ASN1_INTEGER_get(pss->trailerField) != 1) {
613     OPENSSL_PUT_ERROR(EVP, rsa_pss_to_ctx, EVP_R_INVALID_TRAILER);
614     goto err;
615   }
616 
617   if (!EVP_DigestVerifyInit(ctx, &pkctx, md, NULL, pkey) ||
618       !EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PSS_PADDING) ||
619       !EVP_PKEY_CTX_set_rsa_pss_saltlen(pkctx, saltlen) ||
620       !EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md)) {
621     goto err;
622   }
623 
624   ret = 1;
625 
626 err:
627   RSA_PSS_PARAMS_free(pss);
628   if (maskHash) {
629     X509_ALGOR_free(maskHash);
630   }
631   return ret;
632 }
633 
634 /* Customised RSA AlgorithmIdentifier handling. This is called when a signature
635  * is encountered requiring special handling. We currently only handle PSS. */
rsa_digest_verify_init_from_algorithm(EVP_MD_CTX * ctx,X509_ALGOR * sigalg,EVP_PKEY * pkey)636 static int rsa_digest_verify_init_from_algorithm(EVP_MD_CTX *ctx,
637                                                  X509_ALGOR *sigalg,
638                                                  EVP_PKEY *pkey) {
639   /* Sanity check: make sure it is PSS */
640   if (OBJ_obj2nid(sigalg->algorithm) != NID_rsassaPss) {
641     OPENSSL_PUT_ERROR(EVP, rsa_digest_verify_init_from_algorithm,
642                       EVP_R_UNSUPPORTED_SIGNATURE_TYPE);
643     return 0;
644   }
645   return rsa_pss_to_ctx(ctx, sigalg, pkey);
646 }
647 
rsa_digest_sign_algorithm(EVP_MD_CTX * ctx,X509_ALGOR * sigalg)648 static evp_digest_sign_algorithm_result_t rsa_digest_sign_algorithm(
649     EVP_MD_CTX *ctx, X509_ALGOR *sigalg) {
650   int pad_mode;
651   EVP_PKEY_CTX *pkctx = ctx->pctx;
652   if (!EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode)) {
653     return EVP_DIGEST_SIGN_ALGORITHM_ERROR;
654   }
655   if (pad_mode == RSA_PKCS1_PSS_PADDING) {
656     ASN1_STRING *os1 = rsa_ctx_to_pss(pkctx);
657     if (!os1) {
658       return EVP_DIGEST_SIGN_ALGORITHM_ERROR;
659     }
660     X509_ALGOR_set0(sigalg, OBJ_nid2obj(NID_rsassaPss), V_ASN1_SEQUENCE, os1);
661     return EVP_DIGEST_SIGN_ALGORITHM_SUCCESS;
662   }
663 
664   /* Other padding schemes use the default behavior. */
665   return EVP_DIGEST_SIGN_ALGORITHM_DEFAULT;
666 }
667 
668 const EVP_PKEY_ASN1_METHOD rsa_asn1_meth = {
669   EVP_PKEY_RSA,
670   EVP_PKEY_RSA,
671   ASN1_PKEY_SIGPARAM_NULL,
672 
673   "RSA",
674   "OpenSSL RSA method",
675 
676   rsa_pub_decode,
677   rsa_pub_encode,
678   rsa_pub_cmp,
679   rsa_pub_print,
680 
681   rsa_priv_decode,
682   rsa_priv_encode,
683   rsa_priv_print,
684 
685   rsa_opaque,
686   rsa_supports_digest,
687 
688   int_rsa_size,
689   rsa_bits,
690 
691   0,0,0,0,0,0,
692 
693   rsa_sig_print,
694   int_rsa_free,
695 
696   old_rsa_priv_decode,
697   old_rsa_priv_encode,
698 
699   rsa_digest_verify_init_from_algorithm,
700   rsa_digest_sign_algorithm,
701 };
702