rfbssl_gnutls.c revision a430b2b5ca4f0967836f5820e8f03adc17fc0a24 
1/*
2 * rfbssl_gnutls.c - Secure socket funtions (gnutls version)
3 */
4
5/*
6 *  Copyright (C) 2011 Gernot Tenchio
7 *
8 *  This is free software; you can redistribute it and/or modify
9 *  it under the terms of the GNU General Public License as published by
10 *  the Free Software Foundation; either version 2 of the License, or
11 *  (at your option) any later version.
12 *
13 *  This software is distributed in the hope that it will be useful,
14 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
15 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 *  GNU General Public License for more details.
17 *
18 *  You should have received a copy of the GNU General Public License
19 *  along with this software; if not, write to the Free Software
20 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
21 *  USA.
22 */
23
24#include "rfbssl.h"
25#include <gnutls/gnutls.h>
26#include <errno.h>
27
28struct rfbssl_ctx {
29    char peekbuf[2048];
30    int peeklen;
31    int peekstart;
32    gnutls_session_t session;
33    gnutls_certificate_credentials_t x509_cred;
34    gnutls_dh_params_t dh_params;
35#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
36    gnutls_rsa_params_t rsa_params;
37#endif
38};
39
40void rfbssl_log_func(int level, const char *msg)
41{
42    rfbErr("SSL: %s", msg);
43}
44
45static void rfbssl_error(const char *msg, int e)
46{
47    rfbErr("%s: %s (%ld)\n", msg, gnutls_strerror(e), e);
48}
49
50static int rfbssl_init_session(struct rfbssl_ctx *ctx, int fd)
51{
52    gnutls_session_t session;
53    int ret;
54
55    if (!GNUTLS_E_SUCCESS == (ret = gnutls_init(&session, GNUTLS_SERVER))) {
56      /* */
57    } else if (!GNUTLS_E_SUCCESS == (ret = gnutls_priority_set_direct(session, "EXPORT", NULL))) {
58      /* */
59    } else if (!GNUTLS_E_SUCCESS == (ret = gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, ctx->x509_cred))) {
60      /* */
61    } else {
62      gnutls_session_enable_compatibility_mode(session);
63      gnutls_transport_set_ptr(session, (gnutls_transport_ptr_t)(uintptr_t)fd);
64      ctx->session = session;
65    }
66    return ret;
67}
68
69static int generate_dh_params(struct rfbssl_ctx *ctx)
70{
71    int ret;
72    if (GNUTLS_E_SUCCESS == (ret = gnutls_dh_params_init(&ctx->dh_params)))
73	ret = gnutls_dh_params_generate2(ctx->dh_params, 1024);
74    return ret;
75}
76
77#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
78static int generate_rsa_params(struct rfbssl_ctx *ctx)
79{
80    int ret;
81    if (GNUTLS_E_SUCCESS == (ret = gnutls_rsa_params_init(&ctx->rsa_params)))
82	ret = gnutls_rsa_params_generate2(ctx->rsa_params, 512);
83    return ret;
84}
85#endif
86
87struct rfbssl_ctx *rfbssl_init_global(char *key, char *cert)
88{
89    int ret = GNUTLS_E_SUCCESS;
90    struct rfbssl_ctx *ctx = NULL;
91
92    if (NULL == (ctx = malloc(sizeof(struct rfbssl_ctx)))) {
93	ret = GNUTLS_E_MEMORY_ERROR;
94    } else if (!GNUTLS_E_SUCCESS == (ret = gnutls_global_init())) {
95	/* */
96    } else if (!GNUTLS_E_SUCCESS == (ret = gnutls_certificate_allocate_credentials(&ctx->x509_cred))) {
97	/* */
98    } else if ((ret = gnutls_certificate_set_x509_trust_file(ctx->x509_cred, cert, GNUTLS_X509_FMT_PEM)) < 0) {
99	/* */
100    } else if (!GNUTLS_E_SUCCESS == (ret = gnutls_certificate_set_x509_key_file(ctx->x509_cred, cert, key, GNUTLS_X509_FMT_PEM))) {
101	/* */
102    } else if (!GNUTLS_E_SUCCESS == (ret = generate_dh_params(ctx))) {
103	/* */
104#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
105    } else if (!GNUTLS_E_SUCCESS == (ret = generate_rsa_params(ctx))) {
106	/* */
107#endif
108    } else {
109	gnutls_global_set_log_function(rfbssl_log_func);
110	gnutls_global_set_log_level(1);
111	gnutls_certificate_set_dh_params(ctx->x509_cred, ctx->dh_params);
112	return ctx;
113    }
114
115    free(ctx);
116    return NULL;
117}
118
119int rfbssl_init(rfbClientPtr cl)
120{
121    int ret = -1;
122    struct rfbssl_ctx *ctx;
123    char *keyfile;
124    if (!(keyfile = cl->screen->sslkeyfile))
125	keyfile = cl->screen->sslcertfile;
126
127    if (NULL == (ctx = rfbssl_init_global(keyfile,  cl->screen->sslcertfile))) {
128	/* */
129    } else if (GNUTLS_E_SUCCESS != (ret = rfbssl_init_session(ctx, cl->sock))) {
130	/* */
131    } else {
132	while (GNUTLS_E_SUCCESS != (ret = gnutls_handshake(ctx->session))) {
133	    if (ret == GNUTLS_E_AGAIN)
134		continue;
135	    break;
136	}
137    }
138
139    if (ret != GNUTLS_E_SUCCESS) {
140	rfbssl_error(__func__, ret);
141    } else {
142	cl->sslctx = (rfbSslCtx *)ctx;
143	rfbLog("%s protocol initialized\n", gnutls_protocol_get_name(gnutls_protocol_get_version(ctx->session)));
144    }
145    return ret;
146}
147
148static int rfbssl_do_read(rfbClientPtr cl, char *buf, int bufsize)
149{
150    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
151    int ret;
152
153    while ((ret = gnutls_record_recv(ctx->session, buf, bufsize)) < 0) {
154	if (ret == GNUTLS_E_AGAIN) {
155	    /* continue */
156	} else if (ret == GNUTLS_E_INTERRUPTED) {
157	    /* continue */
158	} else {
159	    break;
160	}
161    }
162
163    if (ret < 0) {
164	rfbssl_error(__func__, ret);
165	errno = EIO;
166	ret = -1;
167    }
168
169    return ret < 0 ? -1 : ret;
170}
171
172int rfbssl_write(rfbClientPtr cl, const char *buf, int bufsize)
173{
174    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
175    int ret;
176
177    while ((ret = gnutls_record_send(ctx->session, buf, bufsize)) < 0) {
178	if (ret == GNUTLS_E_AGAIN) {
179	    /* continue */
180	} else if (ret == GNUTLS_E_INTERRUPTED) {
181	    /* continue */
182	} else {
183	    break;
184	}
185    }
186
187    if (ret < 0)
188	rfbssl_error(__func__, ret);
189
190    return ret;
191}
192
193static void rfbssl_gc_peekbuf(struct rfbssl_ctx *ctx, int bufsize)
194{
195    if (ctx->peekstart) {
196	int spaceleft = sizeof(ctx->peekbuf) - ctx->peeklen - ctx->peekstart;
197	if (spaceleft < bufsize) {
198	    memmove(ctx->peekbuf, ctx->peekbuf + ctx->peekstart, ctx->peeklen);
199	    ctx->peekstart = 0;
200	}
201    }
202}
203
204static int __rfbssl_read(rfbClientPtr cl, char *buf, int bufsize, int peek)
205{
206    int ret = 0;
207    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
208
209    rfbssl_gc_peekbuf(ctx, bufsize);
210
211    if (ctx->peeklen) {
212	/* If we have any peek data, simply return that. */
213	ret = bufsize < ctx->peeklen ? bufsize : ctx->peeklen;
214	memcpy (buf, ctx->peekbuf + ctx->peekstart, ret);
215	if (!peek) {
216	    ctx->peeklen -= ret;
217	    if (ctx->peeklen != 0)
218		ctx->peekstart += ret;
219	    else
220		ctx->peekstart = 0;
221	}
222    }
223
224    if (ret < bufsize) {
225	int n;
226	/* read the remaining data */
227	if ((n = rfbssl_do_read(cl, buf + ret, bufsize - ret)) <= 0) {
228	    rfbErr("rfbssl_%s: %s error\n", __func__, peek ? "peek" : "read");
229	    return n;
230	}
231	if (peek) {
232	    memcpy(ctx->peekbuf + ctx->peekstart + ctx->peeklen, buf + ret, n);
233	    ctx->peeklen += n;
234	}
235	ret += n;
236    }
237
238    return ret;
239}
240
241int rfbssl_read(rfbClientPtr cl, char *buf, int bufsize)
242{
243    return __rfbssl_read(cl, buf, bufsize, 0);
244}
245
246int rfbssl_peek(rfbClientPtr cl, char *buf, int bufsize)
247{
248    return __rfbssl_read(cl, buf, bufsize, 1);
249}
250
251int rfbssl_pending(rfbClientPtr cl)
252{
253    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
254    int ret = ctx->peeklen;
255
256    if (ret <= 0)
257	ret = gnutls_record_check_pending(ctx->session);
258
259    return ret;
260}
261
262void rfbssl_destroy(rfbClientPtr cl)
263{
264    struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
265    gnutls_bye(ctx->session, GNUTLS_SHUT_WR);
266    gnutls_deinit(ctx->session);
267    gnutls_certificate_free_credentials(ctx->x509_cred);
268}
269