VirtualBox

source: vbox/trunk/src/VBox/RDP/client-1.8.3/ssl.c@ 67954

Last change on this file since 67954 was 63854, checked in by vboxsync, 8 years ago

RDP/client: fixes for OpenSSL 1.1

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 7.9 KB
Line 
1/* -*- c-basic-offset: 8 -*-
2 rdesktop: A Remote Desktop Protocol client.
3 Secure sockets abstraction layer
4 Copyright (C) Matthew Chapman <matthewc.unsw.edu.au> 1999-2008
5 Copyright (C) Jay Sorg <j@american-data.com> 2006-2008
6
7 This program is free software: you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with this program. If not, see <http://www.gnu.org/licenses/>.
19*/
20
21/*
22 * Oracle GPL Disclaimer: For the avoidance of doubt, except that if any license choice
23 * other than GPL or LGPL is available it will apply instead, Oracle elects to use only
24 * the General Public License version 2 (GPLv2) at this time for any software where
25 * a choice of GPL license versions is made available with the language indicating
26 * that GPLv2 or any later version may be used, or where a choice of which version
27 * of the GPL is applied is otherwise unspecified.
28 */
29
30#include "rdesktop.h"
31#include "ssl.h"
32
33void
34rdssl_sha1_init(RDSSL_SHA1 * sha1)
35{
36 SHA1_Init(sha1);
37}
38
39void
40rdssl_sha1_update(RDSSL_SHA1 * sha1, uint8 * data, uint32 len)
41{
42 SHA1_Update(sha1, data, len);
43}
44
45void
46rdssl_sha1_final(RDSSL_SHA1 * sha1, uint8 * out_data)
47{
48 SHA1_Final(out_data, sha1);
49}
50
51void
52rdssl_md5_init(RDSSL_MD5 * md5)
53{
54 MD5_Init(md5);
55}
56
57void
58rdssl_md5_update(RDSSL_MD5 * md5, uint8 * data, uint32 len)
59{
60 MD5_Update(md5, data, len);
61}
62
63void
64rdssl_md5_final(RDSSL_MD5 * md5, uint8 * out_data)
65{
66 MD5_Final(out_data, md5);
67}
68
69void
70rdssl_rc4_set_key(RDSSL_RC4 * rc4, uint8 * key, uint32 len)
71{
72 RC4_set_key(rc4, len, key);
73}
74
75void
76rdssl_rc4_crypt(RDSSL_RC4 * rc4, uint8 * in_data, uint8 * out_data, uint32 len)
77{
78 RC4(rc4, len, in_data, out_data);
79}
80
81static void
82reverse(uint8 * p, int len)
83{
84 int i, j;
85 uint8 temp;
86
87 for (i = 0, j = len - 1; i < j; i++, j--)
88 {
89 temp = p[i];
90 p[i] = p[j];
91 p[j] = temp;
92 }
93}
94
95void
96rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * modulus,
97 uint8 * exponent)
98{
99#if OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER)
100 BN_CTX *ctx;
101 BIGNUM *mod, *exp, *x, *y;
102 uint8 inr[SEC_MAX_MODULUS_SIZE];
103 int outlen;
104
105 reverse(modulus, modulus_size);
106 reverse(exponent, SEC_EXPONENT_SIZE);
107 memcpy(inr, in, len);
108 reverse(inr, len);
109
110 ctx = BN_CTX_new();
111 mod = BN_new();
112 exp = BN_new();
113 x = BN_new();
114 y = BN_new();
115
116 BN_bin2bn(modulus, modulus_size, mod);
117 BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
118 BN_bin2bn(inr, len, x);
119 BN_mod_exp(y, x, exp, mod, ctx);
120 outlen = BN_bn2bin(y, out);
121 reverse(out, outlen);
122 if (outlen < (int) modulus_size)
123 memset(out + outlen, 0, modulus_size - outlen);
124
125 BN_free(y);
126 BN_clear_free(x);
127 BN_free(exp);
128 BN_free(mod);
129 BN_CTX_free(ctx);
130#else /* OPENSSL_VERSION_NUMBER < 0x10100000 || defined(LIBRESSL_VERSION_NUMBER) */
131 BN_CTX *ctx;
132 BIGNUM mod, exp, x, y;
133 uint8 inr[SEC_MAX_MODULUS_SIZE];
134 int outlen;
135
136 reverse(modulus, modulus_size);
137 reverse(exponent, SEC_EXPONENT_SIZE);
138 memcpy(inr, in, len);
139 reverse(inr, len);
140
141 ctx = BN_CTX_new();
142 BN_init(&mod);
143 BN_init(&exp);
144 BN_init(&x);
145 BN_init(&y);
146
147 BN_bin2bn(modulus, modulus_size, &mod);
148 BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
149 BN_bin2bn(inr, len, &x);
150 BN_mod_exp(&y, &x, &exp, &mod, ctx);
151 outlen = BN_bn2bin(&y, out);
152 reverse(out, outlen);
153 if (outlen < (int) modulus_size)
154 memset(out + outlen, 0, modulus_size - outlen);
155
156 BN_free(&y);
157 BN_clear_free(&x);
158 BN_free(&exp);
159 BN_free(&mod);
160 BN_CTX_free(ctx);
161#endif /* OPENSSL_VERSION_NUMBER < 0x10100000 || defined(LIBRESSL_VERSION_NUMBER) */
162}
163
164/* returns newly allocated RDSSL_CERT or NULL */
165RDSSL_CERT *
166rdssl_cert_read(uint8 * data, uint32 len)
167{
168 /* this will move the data pointer but we don't care, we don't use it again */
169 return d2i_X509(NULL, (D2I_X509_CONST unsigned char **) &data, len);
170}
171
172void
173rdssl_cert_free(RDSSL_CERT * cert)
174{
175 X509_free(cert);
176}
177
178/* returns newly allocated RDSSL_RKEY or NULL */
179RDSSL_RKEY *
180rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
181{
182 EVP_PKEY *epk = NULL;
183 RDSSL_RKEY *lkey;
184 int nid;
185
186 /* By some reason, Microsoft sets the OID of the Public RSA key to
187 the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
188
189 Kudos to Richard Levitte for the following (. intiutive .)
190 lines of code that resets the OID and let's us extract the key. */
191#if OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER)
192 X509_PUBKEY *x509_pk = X509_get_X509_PUBKEY(cert);
193 X509_ALGOR *algor;
194 const ASN1_OBJECT *alg_obj;
195 X509_PUBKEY_get0_param(NULL, NULL, NULL, &algor, x509_pk);
196 X509_ALGOR_get0(&alg_obj, NULL, NULL, algor);
197 nid = OBJ_obj2nid(alg_obj);
198 if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
199 {
200 DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
201 X509_ALGOR_set0(algor, OBJ_nid2obj(NID_rsaEncryption), 0, NULL);
202 }
203#else /* OPENSSL_VERSION_NUMBER < 0x10100000 || defined(LIBRESSL_VERSION_NUMBER) */
204 nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
205 if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
206 {
207 DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
208 ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
209 cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
210 }
211#endif /* OPENSSL_VERSION_NUMBER < 0x10100000 || && defined(LIBRESSL_VERSION_NUMBER) */
212 epk = X509_get_pubkey(cert);
213 if (NULL == epk)
214 {
215 error("Failed to extract public key from certificate\n");
216 return NULL;
217 }
218
219 lkey = RSAPublicKey_dup(EVP_PKEY_get1_RSA(epk));
220 EVP_PKEY_free(epk);
221 *key_len = RSA_size(lkey);
222 return lkey;
223}
224
225/* returns boolean */
226RD_BOOL
227rdssl_certs_ok(RDSSL_CERT * server_cert, RDSSL_CERT * cacert)
228{
229 /* Currently, we don't use the CA Certificate.
230 FIXME:
231 *) Verify the server certificate (server_cert) with the
232 CA certificate.
233 *) Store the CA Certificate with the hostname of the
234 server we are connecting to as key, and compare it
235 when we connect the next time, in order to prevent
236 MITM-attacks.
237 */
238 return True;
239}
240
241int
242rdssl_cert_print_fp(FILE * fp, RDSSL_CERT * cert)
243{
244 return X509_print_fp(fp, cert);
245}
246
247void
248rdssl_rkey_free(RDSSL_RKEY * rkey)
249{
250 RSA_free(rkey);
251}
252
253/* returns error */
254int
255rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, uint8 * modulus,
256 uint32 max_mod_len)
257{
258 int len;
259
260#if OPENSSL_VERSION_NUMBER >= 0x10100000 && !defined(LIBRESSL_VERSION_NUMBER)
261 const BIGNUM *e, *n;
262 RSA_get0_key(rkey, &n, &e, NULL);
263 if ((BN_num_bytes(e) > (int) max_exp_len) ||
264 (BN_num_bytes(n) > (int) max_mod_len))
265 {
266 return 1;
267 }
268 len = BN_bn2bin(e, exponent);
269 reverse(exponent, len);
270 len = BN_bn2bin(n, modulus);
271 reverse(modulus, len);
272#else
273 if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
274 (BN_num_bytes(rkey->n) > (int) max_mod_len))
275 {
276 return 1;
277 }
278 len = BN_bn2bin(rkey->e, exponent);
279 reverse(exponent, len);
280 len = BN_bn2bin(rkey->n, modulus);
281 reverse(modulus, len);
282#endif
283 return 0;
284}
285
286/* returns boolean */
287RD_BOOL
288rdssl_sig_ok(uint8 * exponent, uint32 exp_len, uint8 * modulus, uint32 mod_len,
289 uint8 * signature, uint32 sig_len)
290{
291 /* Currently, we don't check the signature
292 FIXME:
293 */
294 return True;
295}
296
297
298void
299rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
300 unsigned char *md)
301{
302#if OPENSSL_VERSION_NUMBER < 0x10100000 || defined(LIBRESSL_VERSION_NUMBER)
303 HMAC_CTX ctx;
304 HMAC_CTX_init(&ctx);
305#endif
306 HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
307#if OPENSSL_VERSION_NUMBER < 0x10100000 || defined(LIBRESSL_VERSION_NUMBER)
308 HMAC_CTX_cleanup(&ctx);
309#endif
310}
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette