diff --git a/include/sntl/sntllv2.h b/include/sntl/sntllv2.h index 896eb1d..36a027c 100644 --- a/include/sntl/sntllv2.h +++ b/include/sntl/sntllv2.h @@ -74,7 +74,6 @@ typedef struct __connection_t { /* RPC section */ usrtc_t *rpc_list; /** < search tree of possible RPC typed lists */ /* SSL related section */ - SSL_CTX *ctx; /** < SSL context */ SSL *ssl; /** < SSL connection */ int ssl_data_index; /** < SSL index for the custom data */ pthread_mutex_t sslinout[2]; /** < SSL related locks for in and out */ @@ -173,6 +172,7 @@ typedef struct __connections_subsys_type { * during setting up a typed channel */ void (*on_destroy)(conn_t *); /** < callback on connection destroy */ void (*on_pulse)(conn_t *, sexp_t *); /** < callback on pulse emit */ + SSL_CTX *ctx; /** < SSL context */ void *priv; } conn_sys_t; diff --git a/lib/sntllv2.c b/lib/sntllv2.c index dd31962..6af0d36 100644 --- a/lib/sntllv2.c +++ b/lib/sntllv2.c @@ -90,6 +90,9 @@ static int __conn_read(conn_t *co, void *buf, size_t buf_len) do { __try_again: + if(co->flags & SNSX_CLOSED) { + return -1; + } r = SSL_read(co->ssl, buf, (int)buf_len); switch(SSL_get_error (co->ssl, r)) { case SSL_ERROR_NONE: @@ -122,7 +125,10 @@ static int __conn_read(conn_t *co, void *buf, size_t buf_len) fprintf(stderr, "[sntllv2] (RD)SSL connection is cleary closed.\n"); default: __close_conn: + ERR_free_strings(); + co->flags |= SNSX_CLOSED; fprintf(stderr, "[sntllv2] (RD)Unknown error on %s (errno = %d)\n", co->uuid, errno); + ERR_remove_state(0); return -1; } } while(SSL_pending(co->ssl) && !read_blocked); @@ -137,10 +143,12 @@ static int __conn_read(conn_t *co, void *buf, size_t buf_len) if(r < 0) { if(errno == EINTR || errno == EAGAIN) goto __select_retry; fprintf(stderr, "[sntllv2] (RD)Select (%d) on %s\n", errno, co->uuid); + ERR_remove_state(0); return -1; } if(!r) { fprintf(stderr, "[sntllv2] (RD)Nothing to wait for\n"); + ERR_remove_state(0); return 0; } read_blocked = 0; @@ -166,6 +174,9 @@ static int __conn_write(conn_t *co, void *buf, size_t buf_len) fd_set writeset; __retry: + if(co->flags & SNSX_CLOSED) { + return -1; + } r = SSL_write(co->ssl, buf, (int)buf_len); switch(SSL_get_error(co->ssl, r)) { case SSL_ERROR_WANT_READ: @@ -183,7 +194,11 @@ static int __conn_write(conn_t *co, void *buf, size_t buf_len) default: __close_conn: if(r < 0) { + /* set closed flag */ + ERR_free_strings(); + co->flags |= SNSX_CLOSED; fprintf(stderr, "[sntllv2] (WR)Unknown error on %s (%d)\n", co->uuid, r); + ERR_remove_state(0); return -1; } else return r; } @@ -296,6 +311,8 @@ static int __verify_certcall(int preverify_ok, X509_STORE_CTX *ctx) if(!depth) { co->pctx->certid = ASN1_INTEGER_get((const ASN1_INTEGER *)X509_get_serialNumber(ctx->current_cert)); + //X509_STORE_CTX_free(ctx); + //X509_free(ctx->current_cert); /* now we're need to check the ssl cert */ if(ssys->validate_sslpem) { if(ssys->validate_sslpem(co)) return 0; @@ -312,19 +329,65 @@ static int __verify_certcall_dummy(int preverify_ok, X509_STORE_CTX *ctx) return preverify_ok; } +static pthread_mutex_t *lock_cs; +static long *lock_count; + +static void pthreads_locking_callback(int mode, int type, const char *file, int line) +{ + if (mode & CRYPTO_LOCK) { + pthread_mutex_lock(&(lock_cs[type])); + lock_count[type]++; + } else { + pthread_mutex_unlock(&(lock_cs[type])); + } +} + +static void pthreads_thread_id(CRYPTO_THREADID *tid) +{ + CRYPTO_THREADID_set_numeric(tid, (unsigned long)pthread_self()); +} + int sntl_init(void) { + int i; + /* init SSL library */ SSL_library_init(); OpenSSL_add_all_algorithms(); - //SSL_load_error_strings(); + SSL_load_error_strings(); ex_ssldata_index = SSL_get_ex_new_index(0, "__ssldata index", NULL, NULL, NULL); + /* here we go - init all */ + lock_cs = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t)); + lock_count = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(long)); + for (i = 0; i < CRYPTO_num_locks(); i++) { + lock_count[i] = 0; + pthread_mutex_init(&(lock_cs[i]), NULL); + } + + CRYPTO_THREADID_set_callback(pthreads_thread_id); + CRYPTO_set_locking_callback(pthreads_locking_callback); + return 0; } +void sntl_finalize(void) +{ + int i; + + CRYPTO_set_locking_callback(NULL); + + for (i = 0; i < CRYPTO_num_locks(); i++) { + pthread_mutex_destroy(&(lock_cs[i])); + } + OPENSSL_free(lock_cs); + OPENSSL_free(lock_count); + + return; +} + conn_t *__connection_minimal_alloc(struct in_addr *addr) { conn_t *co = malloc(sizeof(conn_t)); @@ -479,7 +542,11 @@ static void __connection_destroy(conn_t *co) pthread_mutex_unlock(&co->write_pending_lock); } + /* free queue */ + ERR_remove_state(0); + /* update use count */ + usleep(1500); _CONN_NOTINUSE(co); /* ok, let's free other if we can */ @@ -513,12 +580,29 @@ static void __connection_destroy(conn_t *co) if(co->pctx->login) free(co->pctx->login); if(co->pctx->passwd) free(co->pctx->passwd); - while((fd = SSL_shutdown(co->ssl)) != 1) { + /*while((fd = SSL_shutdown(co->ssl)) != 1) { if(fd < 0) break; - } + }*/ + SSL_set_shutdown(co->ssl, SSL_RECEIVED_SHUTDOWN | SSL_SENT_SHUTDOWN); + fd = SSL_get_fd(co->ssl); + //int ti = CRYPTO_add(&co->ssl->references, -1, CRYPTO_LOCK_SSL); + //printf("ti = %d;\n", ti); SSL_free(co->ssl); - SSL_CTX_free(co->ctx); + //SSL_CTX_free(co->ssys->ctx); + co->ssl = NULL; + + ERR_remove_thread_state(0); + ERR_remove_state(0); + + + //ENGINE_cleanup(); + //CRYPTO_cleanup_all_ex_data(); + ERR_free_strings(); + //ERR_remove_state(0); + //EVP_cleanup(); + // ENGINE_cleanup(); + close(fd); __connection_second_free(co); __connection_minimal_free(co); @@ -573,9 +657,9 @@ static void *__sntll_thread(void *b) */ while(1) { __again: - if(co->flags & SNSX_CLOSED) goto __finish; /* go away if required asap */ + //if(co->flags & SNSX_CLOSED) goto __finish; /* go away if required asap */ /* works with pending messages */ - if(co->pending_messages) { + if(co->pending_messages && !(co->flags & SNSX_CLOSED)) { pthread_mutex_lock(&co->write_pending_lock); list_for_each_safe(&co->write_pending, iter, siter) { ppm = container_of(iter, ppmsg_t, node); @@ -862,48 +946,55 @@ conn_t *connection_master_link(conn_sys_t *ssys, int sck, struct in_addr *addr) /* ok, now we need to init ssl stuff */ co->ssys = ssys; - /* init SSL certificates and context */ - co->ctx = SSL_CTX_new(TLSv1_2_server_method()); - if(!co->ctx) { r = SNE_ENOMEM; goto __fail; } - else { - /* set verify context */ - SSL_CTX_set_verify(co->ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, - __verify_certcall); - /* set verify depth */ - SSL_CTX_set_verify_depth(co->ctx, VERIFY_DEPTH); - } + /* check up - do we need to initialize SSL context? */ + if(!ssys->ctx) { + /* init SSL certificates and context */ + ssys->ctx = SSL_CTX_new(TLSv1_2_server_method()); + if(!ssys->ctx) { r = SNE_ENOMEM; goto __fail; } + else { + /* set verify context */ + SSL_CTX_set_verify(ssys->ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, + __verify_certcall); + /* set verify depth */ + SSL_CTX_set_verify_depth(ssys->ctx, VERIFY_DEPTH); + + /* set cache policy */ + SSL_CTX_set_session_cache_mode(ssys->ctx, SSL_SESS_CACHE_OFF); + SSL_CTX_set_mode(ssys->ctx, SSL_MODE_RELEASE_BUFFERS); + } + + /* load certificates */ + SSL_CTX_load_verify_locations(ssys->ctx, ssys->rootca, NULL); + /* set the local certificate from CertFile */ + if(SSL_CTX_use_certificate_file(ssys->ctx, ssys->certpem, + SSL_FILETYPE_PEM)<=0) { + r = SNE_ESSL; + goto __fail; + } + /* set the private key from KeyFile (may be the same as CertFile) */ + if(SSL_CTX_use_PrivateKey_file(ssys->ctx, ssys->certkey, + SSL_FILETYPE_PEM)<=0) { + r = SNE_ESSL; + goto __fail; + } + /* verify private key */ + if (!SSL_CTX_check_private_key(ssys->ctx)) { + r = SNE_ESSL; + goto __fail; + } - /* load certificates */ - SSL_CTX_load_verify_locations(co->ctx, ssys->rootca, NULL); - /* set the local certificate from CertFile */ - if(SSL_CTX_use_certificate_file(co->ctx, ssys->certpem, - SSL_FILETYPE_PEM)<=0) { - ERR_print_errors_fp(stderr); - r = SNE_ESSL; - goto __fail; - } - /* set the private key from KeyFile (may be the same as CertFile) */ - if(SSL_CTX_use_PrivateKey_file(co->ctx, ssys->certkey, - SSL_FILETYPE_PEM)<=0) { - r = SNE_ESSL; - goto __fail; - } - /* verify private key */ - if (!SSL_CTX_check_private_key(co->ctx)) { - r = SNE_ESSL; - goto __fail; } /* now we will create an SSL connection */ - co->ssl = SSL_new(co->ctx); + co->ssl = SSL_new(ssys->ctx); if(!co->ssl) { r = SNE_ENOMEM; goto __fail; } else SSL_set_fd(co->ssl, sck); /* attach connected socket */ - SSL_set_accept_state(co->ssl); /* set the context to verify ssl connection */ SSL_set_ex_data(co->ssl, ex_ssldata_index, (void *)co); SSL_set_accept_state(co->ssl); - if(SSL_accept(co->ssl) == -1) { r = SNE_EPERM; goto __fail; } + // if(SSL_accept(co->ssl) == -1) { r = SNE_EPERM; goto __fail; } /* leak here ? */ + SSL_do_handshake(co->ssl); /* ok, now we are able to allocate and so on */ /* set connection to the batch mode */ @@ -985,9 +1076,12 @@ conn_t *connection_master_link(conn_sys_t *ssys, int sck, struct in_addr *addr) for(i = 0; i < MAX_SNTLLTHREADS; i++) { if(bundle == (void *)0xdead) bundle = __sntll_bundle_create(co); if(!bundle) goto __fail5; - r = pthread_create(&co->thrd_poll[i], NULL, __sntll_thread, bundle); + r = pthread_create(&co->thrd_poll[i], NULL, __sntll_thread, bundle); /* and here, alloc tls */ if(r) goto __fail5; - else bundle = (void *)0xdead; + else { + bundle = (void *)0xdead; + pthread_detach(co->thrd_poll[i]); + } } /* all is done, connection now ready */ @@ -996,6 +1090,9 @@ conn_t *connection_master_link(conn_sys_t *ssys, int sck, struct in_addr *addr) r = SNE_SUCCESS; errno = r; + /* free context for this thread */ + ERR_remove_state(0); + return co; __fail5: @@ -1012,7 +1109,6 @@ conn_t *connection_master_link(conn_sys_t *ssys, int sck, struct in_addr *addr) __fail: if(co) { if(co->ssl) SSL_free(co->ssl); - if(co->ctx) SSL_CTX_free(co->ctx); __connection_minimal_free(co); } close(sck); @@ -1052,36 +1148,56 @@ conn_t *connection_link(conn_sys_t *ssys, const char *host, /* ok, now we need to init ssl stuff */ co->ssys = ssys; - /* init SSL certificates and context */ - co->ctx = SSL_CTX_new(TLSv1_2_client_method()); - if(!co->ctx) { r = SNE_ENOMEM; goto __fail; } - else { - /* set verify context */ - SSL_CTX_set_verify(co->ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, - __verify_certcall_dummy); - /* set verify depth */ - SSL_CTX_set_verify_depth(co->ctx, VERIFY_DEPTH); - } + /* check up ssl context */ + if(!ssys->ctx) { + /* init SSL certificates and context */ + ssys->ctx = SSL_CTX_new(TLSv1_2_client_method()); + if(!ssys->ctx) { r = SNE_ENOMEM; goto __fail; } + else { + /* set verify context */ + SSL_CTX_set_verify(ssys->ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, + __verify_certcall_dummy); + /* set verify depth */ + SSL_CTX_set_verify_depth(ssys->ctx, VERIFY_DEPTH); + } - /* load certificates */ - SSL_CTX_load_verify_locations(co->ctx, ssys->rootca, NULL); - /* set the local certificate from CertFile */ - if(SSL_CTX_use_certificate_file(co->ctx, SSL_cert, - SSL_FILETYPE_PEM)<=0) { - ERR_print_errors_fp(stderr); - r = SNE_ESSL; - goto __fail; - } - /* set the private key from KeyFile (may be the same as CertFile) */ - if(SSL_CTX_use_PrivateKey_file(co->ctx, SSL_cert, - SSL_FILETYPE_PEM)<=0) { - r = SNE_ESSL; - goto __fail; - } - /* verify private key */ - if (!SSL_CTX_check_private_key(co->ctx)) { - r = SNE_ESSL; - goto __fail; + /* load certificates */ + SSL_CTX_load_verify_locations(ssys->ctx, ssys->rootca, NULL); + /* set the local certificate from CertFile */ + if(SSL_CTX_use_certificate_file(ssys->ctx, SSL_cert, + SSL_FILETYPE_PEM)<=0) { + r = SNE_ESSL; + goto __fail; + } + /* set the private key from KeyFile (may be the same as CertFile) */ + if(SSL_CTX_use_PrivateKey_file(ssys->ctx, SSL_cert, + SSL_FILETYPE_PEM)<=0) { + r = SNE_ESSL; + goto __fail; + } + /* verify private key */ + if (!SSL_CTX_check_private_key(ssys->ctx)) { + r = SNE_ESSL; + goto __fail; + } + } else { + /* set the local certificate from CertFile */ + if(SSL_CTX_use_certificate_file(ssys->ctx, SSL_cert, + SSL_FILETYPE_PEM)<=0) { + r = SNE_ESSL; + goto __fail; + } + /* set the private key from KeyFile (may be the same as CertFile) */ + if(SSL_CTX_use_PrivateKey_file(ssys->ctx, SSL_cert, + SSL_FILETYPE_PEM)<=0) { + r = SNE_ESSL; + goto __fail; + } + /* verify private key */ + if (!SSL_CTX_check_private_key(ssys->ctx)) { + r = SNE_ESSL; + goto __fail; + } } /* resolve host */ @@ -1102,7 +1218,6 @@ conn_t *connection_link(conn_sys_t *ssys, const char *host, /* create a socket */ sck = socket(PF_INET, SOCK_STREAM, 0); memset(&addr, 0, sizeof(addr)); - // bzero(&addr, sizeof(addr)); /* try to connect it */ addr.sin_family = AF_INET; @@ -1116,7 +1231,7 @@ conn_t *connection_link(conn_sys_t *ssys, const char *host, } /* SSL handshake */ - co->ssl = SSL_new(co->ctx); /* TODO: checkout for it */ + co->ssl = SSL_new(ssys->ctx); /* TODO: checkout for it */ SSL_set_fd(co->ssl, sck); /* attach connected socket */ SSL_set_connect_state(co->ssl); if(SSL_connect(co->ssl) == -1) { @@ -1232,7 +1347,6 @@ conn_t *connection_link(conn_sys_t *ssys, const char *host, __fail: if(co) { if(co->ssl) SSL_free(co->ssl); - if(co->ctx) SSL_CTX_free(co->ctx); __connection_minimal_free(co); } errno = r; @@ -1254,5 +1368,8 @@ int connection_close(conn_t *co) /* we will not wait anything */ co->flags |= SNSX_CLOSED; + /* TODO: wait until all threads will finish */ + usleep(20000); + return SNE_SUCCESS; } diff --git a/tests/lv2ftpc.c b/tests/lv2ftpc.c index 01a8294..400f1ec 100644 --- a/tests/lv2ftpc.c +++ b/tests/lv2ftpc.c @@ -156,6 +156,53 @@ int msg_send(chnl_t *ch, const char *mmbuf, size_t buflen, sxmsg_t *msg, char ** return mr; } +static pthread_mutex_t *lockarray; + +static void lock_callback(int mode, int type, char *file, int line) +{ + (void)file; + (void)line; + if (mode & CRYPTO_LOCK) { + pthread_mutex_lock(&(lockarray[type])); + } + else { + pthread_mutex_unlock(&(lockarray[type])); + } +} + +static unsigned long thread_id(void) +{ + unsigned long ret; + + ret=(unsigned long)pthread_self(); + return(ret); +} + +static void init_locks(void) +{ + int i; + + lockarray=(pthread_mutex_t *)OPENSSL_malloc(CRYPTO_num_locks() * + sizeof(pthread_mutex_t)); + for (i=0; iclient; + conn_sys_t *ssys = s->ssys; + conn_t *co = connection_master_link(ssys, client, NULL); + + if(!co) { + fprintf(stderr, "Cannot create connetion (%d)\n", errno); + } + + return co; +} diff --git a/tests/lv2sd.c b/tests/lv2sd.c index feb455b..0adcdcf 100644 --- a/tests/lv2sd.c +++ b/tests/lv2sd.c @@ -268,8 +268,9 @@ int main(int argc, char **argv) int client = accept(srv, (struct sockaddr*)&addr, &len); /* accept connection as usual */ co = connection_master_link(ssys, client, NULL); /* create connection, that's all */ if(!co) { - fprintf(stderr, "Cannot create connetion (%d)\n", opt); + fprintf(stderr, "Cannot create connetion (%d)\n", errno); } + ERR_remove_state(0); } return 0;