diff options
| -rw-r--r-- | Makefile.am | 2 | ||||
| -rw-r--r-- | PICA_client.c | 242 | ||||
| -rw-r--r-- | PICA_client.h | 26 | ||||
| -rw-r--r-- | PICA_common.h | 8 | ||||
| -rw-r--r-- | PICA_media.c | 640 | ||||
| -rw-r--r-- | PICA_media.h | 177 | ||||
| -rw-r--r-- | PICA_netconf.c | 10 | ||||
| -rw-r--r-- | PICA_node.c | 4 | ||||
| -rw-r--r-- | PICA_node.h | 1 | ||||
| -rw-r--r-- | doc/proto-doc-latest.txt | 74 | ||||
| -rw-r--r-- | pica-client/audiodevice.cpp | 51 | ||||
| -rw-r--r-- | pica-client/audiodevice.h | 23 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.cpp | 31 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.h | 2 | ||||
| -rw-r--r-- | pica-client/callwindow.cpp | 15 | ||||
| -rw-r--r-- | pica-client/callwindow.h | 6 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.cpp | 8 | ||||
| -rw-r--r-- | pica-client/pica-client.pro | 2 | ||||
| -rw-r--r-- | pica-client/skynet.cpp | 95 | ||||
| -rw-r--r-- | pica-client/skynet.h | 19 | ||||
| -rw-r--r-- | pica-client/videodevice.cpp | 87 | ||||
| -rw-r--r-- | pica-client/videodevice.h | 21 |
22 files changed, 1465 insertions, 79 deletions
diff --git a/Makefile.am b/Makefile.am index fa97f67..bc36bdb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -123,7 +123,9 @@ pica-client/dialogs/settingsdialog.h \ pica-client/dialogs/nodesdialog.cpp \ pica-client/dialogs/nodesdialog.h \ PICA_client.c \ +PICA_media.c \ PICA_client.h \ +PICA_media.h \ PICA_msgproc.c \ PICA_msgproc.h \ PICA_common.h PICA_proto.h \ diff --git a/PICA_client.c b/PICA_client.c index cc26b25..c8a873e 100644 --- a/PICA_client.c +++ b/PICA_client.c @@ -16,6 +16,7 @@ */ #include "PICA_security.h" #include "PICA_client.h" +#include "PICA_media.h" #include "PICA_proto.h" #include "PICA_msgproc.h" #include "PICA_common.h" @@ -84,6 +85,8 @@ static unsigned int procmsg_CALL_AUDIOPARAM(unsigned char*, unsigned int, void*) static unsigned int procmsg_CALL_VIDEOPARAM(unsigned char*, unsigned int, void*); static unsigned int procmsg_CALL_AUDIO_PACKET(unsigned char*, unsigned int, void*); static unsigned int procmsg_CALL_VIDEO_PACKET(unsigned char*, unsigned int, void*); +static unsigned int procmsg_MEDIA_PINGREQ(unsigned char*, unsigned int, void*); +static unsigned int procmsg_MEDIA_PINGREP(unsigned char*, unsigned int, void*); static struct PICA_proto_msg* c2n_writebuf_push(struct PICA_c2n *ci, unsigned int msgid, unsigned int size); static struct PICA_proto_msg* c2c_writebuf_push(struct PICA_c2c *chn, unsigned int msgid, unsigned int size); @@ -142,6 +145,53 @@ struct PICA_msginfo c2c_init_messages[] = }; +/* What a datagram of a mediac2c connection can carry, see PICA_media.h */ +const struct PICA_msginfo media_messages[] = +{ + {PICA_PROTO_CALL_AUDIO_PACKET, PICA_MSG_VAR_SIZE, PICA_MSG_VARSIZE_INT16, procmsg_CALL_AUDIO_PACKET}, + {PICA_PROTO_CALL_VIDEO_PACKET, PICA_MSG_VAR_SIZE, PICA_MSG_VARSIZE_INT16, procmsg_CALL_VIDEO_PACKET}, + {PICA_PROTO_PINGREQ, PICA_MSG_FIXED_SIZE, PICA_PROTO_PINGREQ_SIZE, procmsg_MEDIA_PINGREQ}, + {PICA_PROTO_PINGREP, PICA_MSG_FIXED_SIZE, PICA_PROTO_PINGREP_SIZE, procmsg_MEDIA_PINGREP} +}; + +unsigned int PICA_media_process_messages(unsigned char *buf, unsigned int *read_pos, struct PICA_c2c *c2c) +{ + return PICA_processdatastream(buf, read_pos, c2c, media_messages, MSGINFO_MSGSNUM(media_messages)); +} + +void PICA_media_transport_changed(struct PICA_c2c *c2c, int transport, const char *ciphersuitename) +{ + if (callbacks.call_media_transport_cb) + callbacks.call_media_transport_cb(c2c->peer_id, transport, ciphersuitename, + PICA_media_max_payload(c2c)); +} + +int PICA_media_send_ping(struct PICA_c2c *c2c, int reply) +{ + unsigned char buf[PICA_PROTO_PINGREQ_SIZE]; + + buf[0] = buf[1] = reply ? PICA_PROTO_PINGREP : PICA_PROTO_PINGREQ; + RAND_pseudo_bytes(buf + 2, 2); + + return PICA_media_send(c2c, buf, sizeof(buf)); +} + +static unsigned int procmsg_MEDIA_PINGREQ(unsigned char* buf, unsigned int nb, void* p) +{ + PICA_media_send_ping((struct PICA_c2c *)p, 1); + + return 1; +} + +static unsigned int procmsg_MEDIA_PINGREP(unsigned char* buf, unsigned int nb, void* p) +{ + /* Nothing to do: a ping reply is only interesting as a sign that the + * connection still works, and receiving it already updated the time the + * last datagram arrived at. + */ + return 1; +} + int PICA_get_id_from_cert_file(const char *cert_file, unsigned char *id) { X509 *x; @@ -187,7 +237,7 @@ int PICA_get_id_from_cert_string(const char *cert_pem, unsigned char *id) return PICA_id_from_X509(x, id); } -static int verify_callback(int preverify_ok, X509_STORE_CTX *ctx) +int PICA_verify_callback(int preverify_ok, X509_STORE_CTX *ctx) { //return 1 for self-signed certificates char buf[256]; @@ -342,7 +392,7 @@ static int c2c_stage5_starttls(struct PICA_c2c *chnl) return PICA_ERRSSL; } - SSL_set_verify(chnl->ssl, SSL_VERIFY_PEER, verify_callback); + SSL_set_verify(chnl->ssl, SSL_VERIFY_PEER, PICA_verify_callback); SSL_set_verify_depth(chnl->ssl, 1);//peer certificate and one CA if (chnl->outgoing) @@ -366,7 +416,7 @@ static int c2c_stage5_starttls(struct PICA_c2c *chnl) return PICA_OK; } -static int verify_peer_cert_common(X509 **peer_cert, SSL *ssl, const unsigned char *peer_id) +int PICA_verify_peer_cert_common(X509 **peer_cert, SSL *ssl, const unsigned char *peer_id) { unsigned char idbuf[PICA_ID_SIZE]; EVP_PKEY *pubkey; @@ -426,7 +476,7 @@ static int verify_peer_cert_common(X509 **peer_cert, SSL *ssl, const unsigned ch static int c2c_verify_peer_cert(struct PICA_c2c *chnl) { - return verify_peer_cert_common(&chnl->peer_cert, chnl->ssl, chnl->peer_id); + return PICA_verify_peer_cert_common(&chnl->peer_cert, chnl->ssl, chnl->peer_id); } static int c2c_stage6_sendc2cconnreq(struct PICA_c2c *chnl) @@ -454,6 +504,7 @@ static int c2c_start(struct PICA_c2c *chnl) chnl->sck_data = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); IOCTLSETNONBLOCKINGSOCKET(chnl->sck_data, 1); + SETTCPNODELAY(chnl->sck_data); { struct sockaddr_in addr = chnl->conn->srv_addr; @@ -588,6 +639,7 @@ static int directc2c_connect_next(struct PICA_directc2c *dc2c, struct PICA_c2c * case PICA_PROTO_DIRECTC2C_IPV4: dc2c->sck = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); IOCTLSETNONBLOCKINGSOCKET(dc2c->sck, 1); + SETTCPNODELAY(dc2c->sck); memset(&dc2c->addr, 0, sizeof(dc2c->addr)); dc2c->addr.sin_family = AF_INET; @@ -1412,6 +1464,16 @@ int PICA_new_listener(const struct PICA_acc *acc, const char *public_addr, int p goto error_ret_2; } + /* The UDP port with the same number carries call media, see PICA_media.h. + * Failing to open it is not fatal: direct connections still work and + * calls keep their media on the c2c connection. + */ + if (PICA_media_listener_new(acc, public_port, local_port, &lst->media) != PICA_OK) + { + fprintf(stderr, "failed to open UDP port %i for call media\n", local_port); + lst->media = NULL; + } + return PICA_OK; error_ret_2: //(2) @@ -1470,6 +1532,18 @@ int PICA_open_acc(const char *cert_file, goto error_ret_2; } + /* mediac2c connections, see PICA_media.h. Not fatal if it cannot be + * created - calls then keep their media on the c2c connection. + */ +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + a->dtls_ctx = SSL_CTX_new(DTLS_method()); + + if (a->dtls_ctx) + SSL_CTX_set_min_proto_version(a->dtls_ctx, DTLS1_2_VERSION); +#else + a->dtls_ctx = SSL_CTX_new(DTLSv1_2_method()); +#endif + /* Available since openssl 1.1.0 SSL_CTX_set_security_level(a->ctx, PICA_OPENSSL_SECURUTY_LEVEL); */ @@ -1493,11 +1567,23 @@ int PICA_open_acc(const char *cert_file, goto error_ret_3; } + /* The ciphersuites Pica Pica uses are all DHE, so the DTLS context needs + * the parameters as well or it cannot take the server role. + */ + if (a->dtls_ctx && 1 != SSL_CTX_set_tmp_dh(a->dtls_ctx, dh)) + { + SSL_CTX_free(a->dtls_ctx); + a->dtls_ctx = NULL; + } + DH_free(dh); SSL_CTX_set_options(a->ctx, SSL_OP_SINGLE_DH_USE); SSL_CTX_set_options(a->anon_ctx, SSL_OP_SINGLE_DH_USE); + if (a->dtls_ctx) + SSL_CTX_set_options(a->dtls_ctx, SSL_OP_SINGLE_DH_USE); + if (!PICA_get_id_from_cert_file(cert_file, a->id)) { ret_err = PICA_ERRINVCERT; @@ -1548,9 +1634,34 @@ int PICA_open_acc(const char *cert_file, goto error_ret_3; } + /* The DTLS context authenticates the same way as a directc2c connection, + * with the same certificate, key and ciphersuites. Anything going wrong + * here only costs calls their UDP transport, so it is not reported as an + * error - the media stays on the c2c connection instead. + */ + if (a->dtls_ctx) + { + if (password_cb) + { + SSL_CTX_set_default_passwd_cb(a->dtls_ctx, password_cb); + SSL_CTX_set_default_passwd_cb_userdata(a->dtls_ctx, a->id); + } + + if (1 != SSL_CTX_use_certificate_file(a->dtls_ctx, cert_file, SSL_FILETYPE_PEM) + || 1 != SSL_CTX_use_PrivateKey_file(a->dtls_ctx, pkey_file, SSL_FILETYPE_PEM) + || 1 != SSL_CTX_set_cipher_list(a->dtls_ctx, PICA_TLS_CIPHERLIST)) + { + fprintf(stderr, "failed to set up the DTLS context, calls will not use UDP\n"); + SSL_CTX_free(a->dtls_ctx); + a->dtls_ctx = NULL; + } + } + return PICA_OK; error_ret_3: + if (a->dtls_ctx) + SSL_CTX_free(a->dtls_ctx); SSL_CTX_free(a->anon_ctx);//(3) error_ret_2: //(2) SSL_CTX_free(a->ctx); @@ -1803,6 +1914,8 @@ static unsigned int procmsg_CALLANS(unsigned char* buf, unsigned int nb, void *p chan->call_state = PICA_CALL_STATE_ACTIVE; callbacks.call_picked_up_cb(chan->peer_id); + PICA_media_start(chan); + return 1; } @@ -1835,6 +1948,8 @@ static unsigned int procmsg_CALLFIN(unsigned char* buf, unsigned int nb, void *p chan->call_state = PICA_CALL_STATE_IDLE; + PICA_media_close(chan); + callbacks.call_hangup_cb(chan->peer_id); chan->call_audio_params_are_set = 0; chan->call_video_params_are_set = 0; @@ -1889,33 +2004,41 @@ static unsigned int procmsg_CALL_VIDEOPARAM(unsigned char* buf, unsigned int nb, return 1; } +/* A media packet is dropped rather than treated as a protocol error breaking + * the connection. It can legitimately arrive when the receiving side is not + * ready for it: over a mediac2c connection it can overtake the 0x74/0x75 + * message describing its stream, which travels over the c2c connection, and + * it can also arrive after a call has already been hung up locally. A + * malformed packet is dropped for the same reason - over UDP it says nothing + * about the peer. + */ static unsigned int procmsg_CALL_AUDIO_PACKET(unsigned char *buf, unsigned int nb, void* p) { struct PICA_c2c *chan = (struct PICA_c2c *)p; if (chan->call_state != PICA_CALL_STATE_ACTIVE) { - fprintf(stderr, "unexpected CALL_AUDIO_PACKET, no call in progress"); - return 0; + fprintf(stderr, "unexpected CALL_AUDIO_PACKET, no call in progress\n"); + return 1; } if (!chan->call_audio_params_are_set) { - fprintf(stderr, "no AUDIOPARAM was set, can't process AUDIO_PACKET"); - return 0; + fprintf(stderr, "no AUDIOPARAM was set, can't process AUDIO_PACKET\n"); + return 1; } if (*(uint16_t*)(buf + 2) != (nb - 4)) { - return 0; + return 1; } uint16_t tail_size = *(uint16_t*)(buf + 2); if (tail_size < PICA_PROTO_CALL_PACKET_HDRSIZE) { - return 0; + return 1; } uint16_t seq_num = *(uint16_t*)(buf + 4); @@ -1933,27 +2056,27 @@ static unsigned int procmsg_CALL_VIDEO_PACKET(unsigned char *buf, unsigned int n if (chan->call_state != PICA_CALL_STATE_ACTIVE) { - fprintf(stderr, "unexpected CALL_VIDEO_PACKET, no call in progress"); - return 0; + fprintf(stderr, "unexpected CALL_VIDEO_PACKET, no call in progress\n"); + return 1; } if (!chan->call_video_params_are_set) { - fprintf(stderr, "no VIDEOPARAM was set, can't process VIDEO_PACKET"); - return 0; + fprintf(stderr, "no VIDEOPARAM was set, can't process VIDEO_PACKET\n"); + return 1; } if (*(uint16_t*)(buf + 2) != (nb - 4)) { - return 0; + return 1; } uint16_t tail_size = *(uint16_t*)(buf + 2); if (tail_size < PICA_PROTO_CALL_PACKET_HDRSIZE) { - return 0; + return 1; } uint16_t seq_num = *(uint16_t*)(buf + 4); @@ -2096,7 +2219,7 @@ static int directc2c_stage2_starttls(struct PICA_directc2c *d, struct PICA_c2c * d->ssl = SSL_new(c2c->acc->ctx); SSL_set_fd(d->ssl, d->sck); - SSL_set_verify(d->ssl, SSL_VERIFY_PEER, verify_callback); + SSL_set_verify(d->ssl, SSL_VERIFY_PEER, PICA_verify_callback); SSL_set_verify_depth(d->ssl, 1); ret = SSL_connect(d->ssl); @@ -2204,6 +2327,7 @@ int PICA_new_c2n(const struct PICA_acc *acc, const char *nodeaddr, unsigned int } IOCTLSETNONBLOCKINGSOCKET(cid->sck_comm, 1); + SETTCPNODELAY(cid->sck_comm); cid->state = PICA_C2N_STATE_CONNECTING; @@ -2637,7 +2761,7 @@ static struct PICA_c2c * find_matching_c2c(struct PICA_c2n *c2n, struct PICA_dir static int directc2c_verify_peer_cert(struct PICA_directc2c *d, struct PICA_c2c *c2c) { PICA_TRACEFUNC - return verify_peer_cert_common(&d->peer_cert, d->ssl, c2c->peer_id); + return PICA_verify_peer_cert_common(&d->peer_cert, d->ssl, c2c->peer_id); } static void process_directc2c(struct PICA_c2n *c2n, fd_set *rfds, fd_set *wfds) @@ -2658,7 +2782,7 @@ static void process_directc2c(struct PICA_c2n *c2n, fd_set *rfds, fd_set *wfds) { if (d->state == PICA_DIRECTC2C_CONNSTATE_ACTIVE) { - if ((c2c = find_matching_c2c(c2n, d)) && (verify_peer_cert_common(&d->peer_cert, d->ssl, c2c->peer_id) == PICA_OK)) + if ((c2c = find_matching_c2c(c2n, d)) && (PICA_verify_peer_cert_common(&d->peer_cert, d->ssl, c2c->peer_id) == PICA_OK)) { if (c2c->direct) PICA_close_directc2c(c2c->direct); @@ -2807,7 +2931,7 @@ static void process_directc2c(struct PICA_c2n *c2n, fd_set *rfds, fd_set *wfds) } } -static void listener_add_connection(struct PICA_listener *lst, SOCKET s) +static void listener_add_connection(struct PICA_listener *lst, SOCKET s, const struct sockaddr_in *peer_addr) { PICA_TRACEFUNC struct PICA_directc2c *nc; @@ -2817,12 +2941,18 @@ static void listener_add_connection(struct PICA_listener *lst, SOCKET s) nc->is_outgoing = PICA_DIRECTC2C_INCOMING; nc->sck = s; + /* Kept for the media connection: it is the only place the peer's address + * is known on this side, and datagrams arriving from anywhere else + * cannot belong to a call with this peer. The port number is the peer's + * outgoing TCP port and is not used. + */ + nc->addr = *peer_addr; nc->ssl = SSL_new(lst->acc->ctx); nc->state = PICA_DIRECTC2C_CONNSTATE_NEW; SSL_set_fd(nc->ssl, nc->sck); - SSL_set_verify(nc->ssl, SSL_VERIFY_PEER, verify_callback); + SSL_set_verify(nc->ssl, SSL_VERIFY_PEER, PICA_verify_callback); SSL_set_verify_depth(nc->ssl, 1); ret = SSL_accept(nc->ssl); @@ -2906,7 +3036,8 @@ static int process_listener(struct PICA_listener *lst, fd_set *rfds, fd_set *wfd if (s >= 0) { IOCTLSETNONBLOCKINGSOCKET(s, 1); - listener_add_connection(lst, s); + SETTCPNODELAY(s); + listener_add_connection(lst, s, &addr); } } @@ -3029,6 +3160,8 @@ int PICA_event_loop(struct PICA_c2n **connections, int timeout) ic2c = ic2c->next; } + PICA_media_fdset(*ic2n, &rfds, &nfds); + ic2n++; } @@ -3111,6 +3244,9 @@ int PICA_event_loop(struct PICA_c2n **connections, int timeout) { process_directc2c(*ic2n, &rfds, &wfds); } + + //processing media connections of calls in progress + PICA_media_process(*ic2n, &rfds); } ic2n++; } @@ -3659,6 +3795,8 @@ int PICA_pickup_call(struct PICA_c2c *chn) chn->call_state = PICA_CALL_STATE_ACTIVE; + PICA_media_start(chn); + return PICA_OK; } @@ -3703,6 +3841,8 @@ int PICA_hangup_call(struct PICA_c2c *chn) // handle incoming last stream packets? chn->call_state = PICA_CALL_STATE_IDLE; + PICA_media_close(chn); + return PICA_OK; } @@ -3749,37 +3889,41 @@ int PICA_set_call_video_params(struct PICA_c2c *chn, const char *codec_name, uin return PICA_OK; } -int PICA_send_audio_packet(struct PICA_c2c *chn, uint16_t seq_num, uint32_t timestamp, const char *buf, unsigned int len) +static int PICA_send_call_packet(struct PICA_c2c *chn, unsigned int msgid, uint16_t seq_num, + uint32_t timestamp, const char *buf, unsigned int len) { struct PICA_proto_msg *mp; if (chn->call_state != PICA_CALL_STATE_ACTIVE) return PICA_ERRCALLNOTINPROGRESS; - if ((mp = c2c_writebuf_push(chn, PICA_PROTO_CALL_AUDIO_PACKET, len + 4 + PICA_PROTO_CALL_PACKET_HDRSIZE))) - { - *((uint16_t*)mp->tail) = len + PICA_PROTO_CALL_PACKET_HDRSIZE; - *((uint16_t*)(mp->tail + 2)) = seq_num; - *((uint32_t*)(mp->tail + 2 + PICA_PROTO_CALL_PACKET_SEQNUM_SIZE)) = timestamp; - memcpy(mp->tail + 2 + PICA_PROTO_CALL_PACKET_HDRSIZE, buf, len); + if (len > PICA_media_max_payload(chn)) + return PICA_ERRMSGSIZE; - } - else + if (PICA_media_is_active(chn)) { - return PICA_ERRNOMEM; - } + /* Written to the socket here and now instead of being appended to + * the write buffer the event loop drains: a datagram never blocks + * for long, and waiting for the next pass of the event loop would + * add more delay to every packet than the rest of the call path + * costs together. + */ + unsigned char pkt[PICA_MEDIA_LINK_MTU]; + unsigned int pktlen = 4 + PICA_PROTO_CALL_PACKET_HDRSIZE + len; - return PICA_OK; -} + if (pktlen > sizeof(pkt)) + return PICA_ERRMSGSIZE; -int PICA_send_video_packet(struct PICA_c2c *chn, uint16_t seq_num, uint32_t timestamp, const char *buf, unsigned int len) -{ - struct PICA_proto_msg *mp; + pkt[0] = pkt[1] = msgid; + *((uint16_t*)(pkt + 2)) = len + PICA_PROTO_CALL_PACKET_HDRSIZE; + *((uint16_t*)(pkt + 4)) = seq_num; + *((uint32_t*)(pkt + 4 + PICA_PROTO_CALL_PACKET_SEQNUM_SIZE)) = timestamp; + memcpy(pkt + 4 + PICA_PROTO_CALL_PACKET_HDRSIZE, buf, len); - if (chn->call_state != PICA_CALL_STATE_ACTIVE) - return PICA_ERRCALLNOTINPROGRESS; + return PICA_media_send(chn, pkt, pktlen); + } - if ((mp = c2c_writebuf_push(chn, PICA_PROTO_CALL_VIDEO_PACKET, len + 4 + PICA_PROTO_CALL_PACKET_HDRSIZE))) + if ((mp = c2c_writebuf_push(chn, msgid, len + 4 + PICA_PROTO_CALL_PACKET_HDRSIZE))) { *((uint16_t*)mp->tail) = len + PICA_PROTO_CALL_PACKET_HDRSIZE; *((uint16_t*)(mp->tail + 2)) = seq_num; @@ -3795,6 +3939,16 @@ int PICA_send_video_packet(struct PICA_c2c *chn, uint16_t seq_num, uint32_t time return PICA_OK; } +int PICA_send_audio_packet(struct PICA_c2c *chn, uint16_t seq_num, uint32_t timestamp, const char *buf, unsigned int len) +{ + return PICA_send_call_packet(chn, PICA_PROTO_CALL_AUDIO_PACKET, seq_num, timestamp, buf, len); +} + +int PICA_send_video_packet(struct PICA_c2c *chn, uint16_t seq_num, uint32_t timestamp, const char *buf, unsigned int len) +{ + return PICA_send_call_packet(chn, PICA_PROTO_CALL_VIDEO_PACKET, seq_num, timestamp, buf, len); +} + void PICA_close_directc2c(struct PICA_directc2c *d) { PICA_TRACEFUNC @@ -3842,6 +3996,8 @@ void PICA_close_c2c(struct PICA_c2c *chn) //puts("PICA_close_c2c_chkp1");//debug + PICA_media_close(chn); + if (chn -> sendfile_stream) fclose(chn -> sendfile_stream); @@ -3915,6 +4071,10 @@ void PICA_close_acc(struct PICA_acc *a) void PICA_close_listener(struct PICA_listener *l) { CLOSE(l->sck_listener); + + if (l->media) + PICA_media_listener_close(l->media); + free(l->public_addr_dns); while(l->accepted_connections) diff --git a/PICA_client.h b/PICA_client.h index 90e16ef..22acafc 100644 --- a/PICA_client.h +++ b/PICA_client.h @@ -50,6 +50,7 @@ typedef u_short in_port_t; #include <sys/socket.h> #include <sys/ioctl.h> #include <netinet/in.h> +#include <netinet/tcp.h> #include <arpa/inet.h> #include <netdb.h> #include <signal.h> @@ -149,6 +150,8 @@ typedef u_short in_port_t; struct PICA_c2n; struct PICA_c2c; +struct PICA_media_channel; +struct PICA_media_listener; struct PICA_acc { @@ -158,6 +161,8 @@ struct PICA_acc unsigned char id[PICA_ID_SIZE]; SSL_CTX* ctx; SSL_CTX* anon_ctx; + /** used for mediac2c connections, see PICA_media.h */ + SSL_CTX* dtls_ctx; const char *pkey_file; const char *cert_file; int (*password_cb)(char *buf, int size, int rwflag, void *userdata); @@ -304,7 +309,7 @@ struct PICA_c2c FILE *recvfile_stream; /** this flag should be set when connection - * must be closed after sending last pushed packet + * must be closed after sending the last pushed packet */ int disconnect_on_empty_write_buf; int switched_to_directc2c_write; @@ -316,6 +321,8 @@ struct PICA_c2c enum PICA_call_state call_state; int call_audio_params_are_set; int call_video_params_are_set; + /** media connection of the call in progress, see PICA_media.h */ + struct PICA_media_channel *media; }; struct PICA_listener @@ -330,6 +337,11 @@ struct PICA_listener const char *public_addr_dns; struct PICA_directc2c *accepted_connections; + + /** listens on the UDP port with the same number, see PICA_media.h. + * NULL if that port could not be opened. + */ + struct PICA_media_listener *media; }; struct PICA_client_callbacks @@ -427,6 +439,18 @@ struct PICA_client_callbacks void (*call_audio_packet_cb)(const unsigned char *peer_id, uint16_t seq_num, uint32_t timestamp, uint16_t size, const char *pkt_data); void (*call_video_packet_cb)(const unsigned char *peer_id, uint16_t seq_num, uint32_t timestamp, uint16_t size, const char *pkt_data); + + /** + * The media packets of the call in progress started or stopped being + * carried over a mediac2c connection. + * \param transport PICA_CALL_TRANSPORT_C2C or PICA_CALL_TRANSPORT_MEDIAC2C + * \param ciphersuitename ciphersuite of the media connection, NULL when + * the transport is not mediac2c + * \param max_payload largest amount of data that can be passed to + * PICA_send_audio_packet()/PICA_send_video_packet() from now on + */ + void (*call_media_transport_cb)(const unsigned char *peer_id, int transport, + const char *ciphersuitename, unsigned int max_payload); }; diff --git a/PICA_common.h b/PICA_common.h index ee1fc54..8f35cf6 100644 --- a/PICA_common.h +++ b/PICA_common.h @@ -21,10 +21,18 @@ #define CLOSE closesocket #define SHUTDOWN(s) shutdown(s,SD_BOTH) #define IOCTLSETNONBLOCKINGSOCKET(s, a) {unsigned long arg = (a); ioctlsocket(s, FIONBIO, (unsigned long*)&arg); } +/* Winsock takes the option value as char*, unlike the BSD sockets API */ +#define SETTCPNODELAY(s) {int arg = 1; setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (const char*)&arg, sizeof(arg)); } #else #define CLOSE close #define SHUTDOWN(s) shutdown(s,SHUT_RDWR) #define IOCTLSETNONBLOCKINGSOCKET(s, a) {int arg = (a); ioctl(s, FIONBIO, (int*)&arg); } +/* Disables Nagle's algorithm. Every Pica Pica protocol message is small and + * is meant to be delivered as soon as it is written - holding one back until + * the previous one is acknowledged adds delay to chat messages and, much more + * noticeably, to the audio and video packets of a call. + */ +#define SETTCPNODELAY(s) {int arg = 1; setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &arg, sizeof(arg)); } #endif #if defined(WIN32) || defined (__APPLE__) diff --git a/PICA_media.c b/PICA_media.c new file mode 100644 index 0000000..b0f33cf --- /dev/null +++ b/PICA_media.c @@ -0,0 +1,640 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#include "PICA_security.h" +#include "PICA_client.h" +#include "PICA_media.h" +#include "PICA_proto.h" +#include "PICA_common.h" +#include <string.h> +#include <time.h> + +#define PICA_DEBUG + +#ifdef PICA_DEBUG +#define PICA_TRACEFUNC fprintf(stderr, "%s()\n", __PRETTY_FUNCTION__); +#else +#define PICA_TRACEFUNC +#endif + +static void media_fail(struct PICA_media_channel *m); +static void media_handshake(struct PICA_media_channel *m); + +static int media_socket_open(SOCKET *s, int local_port) +{ + int flag = 1; + struct sockaddr_in a; + + *s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + + if (*s == SOCKET_ERROR) + return PICA_ERRSOCK; + + setsockopt(*s, SOL_SOCKET, SO_REUSEADDR, (const char*)&flag, sizeof(flag)); + + memset(&a, 0, sizeof(a)); + a.sin_family = AF_INET; + a.sin_addr.s_addr = INADDR_ANY; + /* local_port 0 asks the system for an unused port, which is what the + * connecting side wants - only the accepting side needs a port number + * known to the peer in advance. + */ + a.sin_port = htons(local_port); + + if (bind(*s, (const struct sockaddr*)&a, sizeof(a)) == SOCKET_ERROR) + { + CLOSE(*s); + *s = -1; + return PICA_ERRSOCK; + } + + IOCTLSETNONBLOCKINGSOCKET(*s, 1); + + return PICA_OK; +} + +int PICA_media_listener_new(const struct PICA_acc *acc, int public_port, int local_port, + struct PICA_media_listener **ml) +{ + struct PICA_media_listener *l; + int ret; + + PICA_TRACEFUNC + + if (!acc || !ml || public_port <= 0 || public_port > 65535 || local_port <= 0 || local_port > 65535) + return PICA_ERRINVARG; + + l = *ml = (struct PICA_media_listener*)calloc(sizeof(struct PICA_media_listener), 1); + + if (!l) + return PICA_ERRNOMEM; + + l->acc = acc; + l->public_port = public_port; + l->local_port = local_port; + + ret = media_socket_open(&l->sck, local_port); + + if (ret != PICA_OK) + { + free(l); + *ml = 0; + return ret; + } + + return PICA_OK; +} + +void PICA_media_listener_close(struct PICA_media_listener *ml) +{ + PICA_TRACEFUNC + + if (!ml) + return; + + if (ml->channel) + { + ml->channel->listener = NULL; + ml->channel->sck = -1; + ml->channel = NULL; + } + + if (ml->sck >= 0) + CLOSE(ml->sck); + + free(ml); +} + +/* The listener's socket gets connect()ed to the peer for the duration of a + * call, so that the system drops everything coming from anywhere else. It has + * to be given back in its original, unconnected state afterwards; recreating + * it is the one way of doing that which behaves the same everywhere. + */ +static void media_listener_rearm(struct PICA_media_listener *ml) +{ + PICA_TRACEFUNC + + if (ml->sck >= 0) + CLOSE(ml->sck); + + if (media_socket_open(&ml->sck, ml->local_port) != PICA_OK) + { + fprintf(stderr, "failed to reopen the media listening socket on port %i\n", ml->local_port); + ml->sck = -1; + } +} + +static int media_ssl_new(struct PICA_media_channel *m) +{ + BIO *bio; + + m->ssl = SSL_new(m->c2c->acc->dtls_ctx); + + if (!m->ssl) + return PICA_ERRSSL; + + bio = BIO_new_dgram(m->sck, BIO_NOCLOSE); + + if (!bio) + { + SSL_free(m->ssl); + m->ssl = NULL; + return PICA_ERRSSL; + } + + /* The socket is connected, so the system already knows where the + * datagrams go; the BIO has to be told as well, otherwise it has no + * peer address to hand to sendto(). + */ + BIO_ctrl(bio, BIO_CTRL_DGRAM_SET_CONNECTED, 0, (char*)&m->peer_addr); + + SSL_set_bio(m->ssl, bio, bio); + + SSL_set_verify(m->ssl, SSL_VERIFY_PEER, PICA_verify_callback); + SSL_set_verify_depth(m->ssl, 1); + + /* OpenSSL performs no path MTU discovery of its own: off Linux it cannot + * even read the value the system discovered, and falls back to a 228 + * octet floor that would fragment every handshake message. Telling it + * the MTU explicitly, and stopping it from asking the BIO + * (SSL_OP_NO_QUERY_MTU), gives the same behaviour everywhere. It also + * keeps OpenSSL from shrinking the value on its own after two handshake + * retransmissions, which it never undoes. + * This has to happen before the handshake starts: with + * SSL_OP_NO_QUERY_MTU set and no MTU, writing a handshake message fails. + */ + SSL_set_options(m->ssl, SSL_OP_NO_QUERY_MTU); +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + DTLS_set_link_mtu(m->ssl, PICA_MEDIA_LINK_MTU); +#else + SSL_set_mtu(m->ssl, PICA_MEDIA_LINK_MTU - PICA_MEDIA_IPUDP_OVERHEAD); +#endif + + return PICA_OK; +} + +static struct PICA_media_channel *media_channel_new(struct PICA_c2c *c2c) +{ + struct PICA_media_channel *m; + + m = (struct PICA_media_channel*)calloc(sizeof(struct PICA_media_channel), 1); + + if (!m) + return NULL; + + m->c2c = c2c; + m->sck = -1; + m->started = time(0); + m->last_rx = m->started; + m->last_tx = m->started; + + return m; +} + +void PICA_media_start(struct PICA_c2c *c2c) +{ + struct PICA_media_channel *m; + int ret; + + PICA_TRACEFUNC + + if (c2c->media) + return; + + /* Media over UDP is only attempted between clients that already reached + * each other directly over TCP: that address is known to work and the + * user has already agreed to reveal it by enabling direct connections. + */ + if (c2c->directc2c_state != PICA_DIRECTC2C_STATE_ACTIVE || !c2c->direct) + return; + + if (!c2c->acc->dtls_ctx) + return; + + m = media_channel_new(c2c); + + if (!m) + return; + + if (c2c->direct->is_outgoing == PICA_DIRECTC2C_OUTGOING) + { + /* We reached the peer over TCP at this address, so we are the one + * that can reach it over UDP as well - the peer's announced port + * number is used for both. + */ + m->is_dtls_client = 1; + m->peer_addr = c2c->direct->addr; + + ret = media_socket_open(&m->sck, 0); + + if (ret != PICA_OK) + { + free(m); + return; + } + + m->owns_sck = 1; + + if (connect(m->sck, (struct sockaddr*)&m->peer_addr, sizeof(m->peer_addr)) == SOCKET_ERROR) + { + CLOSE(m->sck); + free(m); + return; + } + + if (media_ssl_new(m) != PICA_OK) + { + CLOSE(m->sck); + free(m); + return; + } + + SSL_set_connect_state(m->ssl); + + m->state = PICA_MEDIA_STATE_HANDSHAKE; + c2c->media = m; + + media_handshake(m); + } + else + { + struct PICA_media_listener *ml; + + if (!c2c->conn->directc2c_listener || !c2c->conn->directc2c_listener->media) + { + free(m); + return; + } + + ml = c2c->conn->directc2c_listener->media; + + if (ml->sck < 0 || ml->channel) + { + free(m); + return; + } + + /* Nothing to do until the peer sends its first datagram: the source + * address it arrives from is the only way to learn which port its + * NAT gave it. + */ + m->listener = ml; + m->sck = ml->sck; + m->owns_sck = 0; + m->state = PICA_MEDIA_STATE_LISTENING; + + ml->channel = m; + c2c->media = m; + } +} + +void PICA_media_close(struct PICA_c2c *c2c) +{ + struct PICA_media_channel *m = c2c->media; + + PICA_TRACEFUNC + + if (!m) + return; + + c2c->media = NULL; + + if (m->peer_cert) + X509_free(m->peer_cert); + + if (m->ssl) + SSL_free(m->ssl); + + if (m->owns_sck && m->sck >= 0) + CLOSE(m->sck); + + if (m->listener) + { + m->listener->channel = NULL; + + /* the socket was connected to the peer while the call lasted */ + if (m->state != PICA_MEDIA_STATE_LISTENING) + media_listener_rearm(m->listener); + } + + free(m); +} + +static void media_fail(struct PICA_media_channel *m) +{ + struct PICA_c2c *c2c = m->c2c; + + PICA_TRACEFUNC + + PICA_media_close(c2c); + + /* the call carries on, its media goes back to the c2c connection */ + PICA_media_transport_changed(c2c, PICA_CALL_TRANSPORT_C2C, NULL); +} + +int PICA_media_is_active(const struct PICA_c2c *c2c) +{ + return c2c->media && c2c->media->state == PICA_MEDIA_STATE_ACTIVE; +} + +unsigned int PICA_media_max_payload(const struct PICA_c2c *c2c) +{ + unsigned int mtu; + + if (!PICA_media_is_active(c2c)) + return PICA_PROTO_C2CMSG_MAXDATASIZE - PICA_PROTO_CALL_PACKET_HDRSIZE; + + mtu = c2c->media->max_payload; + + if (mtu <= 4 + PICA_PROTO_CALL_PACKET_HDRSIZE) + return 0; + + /* the message header, its length field and the media packet header all + * have to fit into the same datagram + */ + return mtu - 4 - PICA_PROTO_CALL_PACKET_HDRSIZE; +} + +int PICA_media_send(struct PICA_c2c *c2c, const unsigned char *buf, unsigned int len) +{ + struct PICA_media_channel *m = c2c->media; + int ret; + + if (!PICA_media_is_active(c2c)) + return PICA_ERRINVARG; + + ret = SSL_write(m->ssl, buf, len); + + if (ret > 0) + { + m->last_tx = time(0); + return PICA_OK; + } + + switch (SSL_get_error(m->ssl, ret)) + { + case SSL_ERROR_WANT_WRITE: + case SSL_ERROR_WANT_READ: + /* The send buffer is full - the packet is dropped rather than + * queued. Delaying live media is worse than losing a frame of it, + * and the receiving side is built to cope with losses anyway. + */ + return PICA_OK; + + default: + media_fail(m); + return PICA_ERRSSL; + } +} + +static void media_activate(struct PICA_media_channel *m) +{ + int ret; + + PICA_TRACEFUNC + + ret = PICA_verify_peer_cert_common(&m->peer_cert, m->ssl, m->c2c->peer_id); + + if (ret != PICA_OK) + { + fprintf(stderr, "media connection peer certificate verification failed: %i\n", ret); + media_fail(m); + return; + } + +#if OPENSSL_VERSION_NUMBER >= 0x10100000L + m->max_payload = (unsigned int)DTLS_get_data_mtu(m->ssl); +#else + /* DTLS record header, explicit nonce and authentication tag of the + * ciphersuites Pica Pica uses + */ + m->max_payload = PICA_MEDIA_LINK_MTU - PICA_MEDIA_IPUDP_OVERHEAD - 13 - 8 - 16; +#endif + + m->state = PICA_MEDIA_STATE_ACTIVE; + m->last_rx = time(0); + + PICA_media_transport_changed(m->c2c, PICA_CALL_TRANSPORT_MEDIAC2C, SSL_get_cipher_name(m->ssl)); +} + +static void media_handshake(struct PICA_media_channel *m) +{ + int ret; + + ret = SSL_do_handshake(m->ssl); + + if (ret == 1) + { + media_activate(m); + return; + } + + switch (SSL_get_error(m->ssl, ret)) + { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + break; + + default: + fprintf(stderr, "media connection DTLS handshake failed\n"); + media_fail(m); + } +} + +/* The first datagram of a call arrives at the listening socket from a source + * port only the peer's NAT knows. Once it is seen, the socket is connected to + * that address for the rest of the call. + */ +static void media_accept(struct PICA_media_channel *m) +{ + struct sockaddr_in from; +#ifdef WIN32 + int fromlen = sizeof(from); +#else + socklen_t fromlen = sizeof(from); +#endif + /* Large enough for any datagram the peer can send us, so that the peek + * never truncates: a truncated peek is reported as an error on Windows. + */ + unsigned char peekbuf[PICA_MEDIA_READBUFSIZE]; + int ret; + + PICA_TRACEFUNC + + ret = recvfrom(m->sck, (char*)peekbuf, sizeof(peekbuf), MSG_PEEK, (struct sockaddr*)&from, &fromlen); + + if (ret < 0) + return; + + /* The peer of the call is already known - it is the client on the other + * end of the directc2c connection - so a datagram from any other address + * cannot belong to this call. Only the address is checked here, the port + * number is whatever the peer's system picked. + */ + if (from.sin_addr.s_addr != m->c2c->direct->addr.sin_addr.s_addr) + { + /* consume it, otherwise it is peeked again on every pass */ + recvfrom(m->sck, (char*)peekbuf, sizeof(peekbuf), 0, (struct sockaddr*)&from, &fromlen); + fprintf(stderr, "dropped a datagram from an address the call is not with\n"); + return; + } + + m->peer_addr = from; + + if (connect(m->sck, (struct sockaddr*)&m->peer_addr, sizeof(m->peer_addr)) == SOCKET_ERROR) + { + media_fail(m); + return; + } + + if (media_ssl_new(m) != PICA_OK) + { + media_fail(m); + return; + } + + SSL_set_accept_state(m->ssl); + + m->state = PICA_MEDIA_STATE_HANDSHAKE; + + media_handshake(m); +} + +static void media_read(struct PICA_media_channel *m) +{ + int ret; + + /* One SSL_read() per datagram, so this loop runs until the socket has + * nothing left for us. + */ + while (1) + { + ret = SSL_read(m->ssl, m->read_buf, PICA_MEDIA_READBUFSIZE); + + if (ret <= 0) + { + switch (SSL_get_error(m->ssl, ret)) + { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + return; + + default: + fprintf(stderr, "media connection read failed\n"); + media_fail(m); + return; + } + } + + m->last_rx = time(0); + m->read_pos = (unsigned int)ret; + + /* A datagram carries exactly one message. Anything that does not + * parse is dropped together with the rest of the datagram - unlike + * on a c2c connection, bad data here says nothing about the peer, + * anyone able to reach the port can send it. + */ + PICA_media_process_messages(m->read_buf, &m->read_pos, m->c2c); + + m->read_pos = 0; + } +} + +void PICA_media_fdset(struct PICA_c2n *c2n, fd_set *rfds, int *nfds) +{ + struct PICA_c2c *c2c = c2n->chan_list_head; + + while (c2c) + { + if (c2c->media && c2c->media->sck >= 0) + { + FD_SET(c2c->media->sck, rfds); + + if (c2c->media->sck > *nfds) + *nfds = c2c->media->sck; + } + + c2c = c2c->next; + } +} + +void PICA_media_process(struct PICA_c2n *c2n, fd_set *rfds) +{ + struct PICA_c2c *chan = c2n->chan_list_head; + time_t now = time(0); + + while (chan) + { + struct PICA_c2c *c2c = chan; + struct PICA_media_channel *m = c2c->media; + + chan = chan->next; + + if (!m) + continue; + + if (m->state != PICA_MEDIA_STATE_ACTIVE + && (now - m->started) > PICA_MEDIA_HANDSHAKE_TIMEOUT) + { + fprintf(stderr, "media connection was not established in time\n"); + media_fail(m); + continue; + } + + /* Anything below can close the media connection, which frees m - + * hence the checks through c2c, which stays valid. + */ + switch (m->state) + { + case PICA_MEDIA_STATE_LISTENING: + if (m->sck >= 0 && FD_ISSET(m->sck, rfds)) + media_accept(m); + break; + + case PICA_MEDIA_STATE_HANDSHAKE: + /* Lost handshake messages are retransmitted by OpenSSL, but + * only when it is told that its timer has expired. + */ + DTLSv1_handle_timeout(m->ssl); + + if (m->sck >= 0 && FD_ISSET(m->sck, rfds)) + media_handshake(m); + break; + + case PICA_MEDIA_STATE_ACTIVE: + if (m->sck >= 0 && FD_ISSET(m->sck, rfds)) + media_read(m); + + if (!PICA_media_is_active(c2c)) + break; + + if ((now - m->last_rx) > PICA_MEDIA_SILENCE_TIMEOUT) + { + fprintf(stderr, "media connection went silent\n"); + media_fail(m); + break; + } + + if ((now - m->last_tx) >= PICA_MEDIA_KEEPALIVE_INTERVAL) + PICA_media_send_ping(c2c, 0); + + break; + + default: + break; + } + } +} diff --git a/PICA_media.h b/PICA_media.h new file mode 100644 index 0000000..971b918 --- /dev/null +++ b/PICA_media.h @@ -0,0 +1,177 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ + +/* mediac2c connections - call media over UDP, protected by DTLS. + * + * A mediac2c connection exists only while a call is in progress and only + * between two clients that already have an established directc2c connection. + * It reuses that connection's address and port number with UDP instead of + * TCP, so nothing has to be announced or negotiated over the c2c connection: + * the client that made the outgoing directc2c connection is the DTLS client, + * the one that accepted it is the DTLS server. + * + * Media packets are sent over this connection whenever it is established and + * over the c2c/directc2c connection otherwise; the receiving side accepts + * them over either transport at any time, so there is no moment where the + * media stops while the transport changes. + */ + +#ifndef PICA_MEDIA_H +#define PICA_MEDIA_H + +#include "PICA_client.h" + +/* Link MTU assumed for a mediac2c connection until path MTU discovery + * (RFC 8899) is implemented. 1200 octets is the value QUIC uses as the size + * every path is expected to carry. + */ +#define PICA_MEDIA_LINK_MTU 1200 +/* IPv4 + UDP headers */ +#define PICA_MEDIA_IPUDP_OVERHEAD 28 + +/* Amount of media data that certainly fits into one datagram of a mediac2c + * connection: the assumed link MTU less a generous allowance for the IP, UDP, + * DTLS record and protocol message headers, whose exact sizes depend on the + * ciphersuite that ends up being negotiated. + * It lets the media be produced in datagram sized pieces from the start of a + * call, before it is known whether a media connection will come up at all, + * rather than having to change size in the middle of one. + */ +#define PICA_MEDIA_SAFE_PAYLOAD (PICA_MEDIA_LINK_MTU - 100) + +/* How long a call keeps trying to bring the media connection up before + * giving up and carrying the media over the c2c connection for the rest of + * the call, in seconds. + */ +#define PICA_MEDIA_HANDSHAKE_TIMEOUT 10 + +/* An established media connection with no datagram received for this many + * seconds is considered broken; media falls back to the c2c connection. + */ +#define PICA_MEDIA_SILENCE_TIMEOUT 10 + +/* Ping is sent over an established media connection after this many seconds + * without anything being sent, to hold NAT and firewall state open. + */ +#define PICA_MEDIA_KEEPALIVE_INTERVAL 4 + +#define PICA_MEDIA_READBUFSIZE 4096 + +/* transport of the media packets of a call, reported by call_media_transport_cb */ +#define PICA_CALL_TRANSPORT_C2C 0 +#define PICA_CALL_TRANSPORT_MEDIAC2C 1 + +enum PICA_media_state +{ + PICA_MEDIA_STATE_NEW = 0, + /* DTLS server role, socket bound, waiting for the first datagram */ + PICA_MEDIA_STATE_LISTENING, + PICA_MEDIA_STATE_HANDSHAKE, + PICA_MEDIA_STATE_ACTIVE, + PICA_MEDIA_STATE_FAILED +}; + +struct PICA_media_listener +{ + const struct PICA_acc *acc; + SOCKET sck; + int public_port; + int local_port; + + /* The media connection currently using this listener's socket, if any. + * Only one call can be in progress at a time, so there is at most one. + */ + struct PICA_media_channel *channel; +}; + +struct PICA_media_channel +{ + struct PICA_c2c *c2c; + /* set when this side has the DTLS server role */ + struct PICA_media_listener *listener; + + SOCKET sck; + /* 0 when sck belongs to the listener and must not be closed with the + * channel + */ + int owns_sck; + struct sockaddr_in peer_addr; + + SSL *ssl; + X509 *peer_cert; + int is_dtls_client; + + enum PICA_media_state state; + time_t started; + time_t last_rx; + time_t last_tx; + + unsigned int max_payload; + + unsigned char read_buf[PICA_MEDIA_READBUFSIZE]; + unsigned int read_pos; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/* implemented in PICA_media.c */ + +int PICA_media_listener_new(const struct PICA_acc *acc, int public_port, int local_port, + struct PICA_media_listener **ml); +void PICA_media_listener_close(struct PICA_media_listener *ml); + +/** + * Brings up a media connection for a call that has just become active. + * Does nothing if the preconditions (an established directc2c connection, + * and a media listener on the accepting side) are not met - the media then + * stays on the c2c connection. + */ +void PICA_media_start(struct PICA_c2c *c2c); +void PICA_media_close(struct PICA_c2c *c2c); + +int PICA_media_is_active(const struct PICA_c2c *c2c); +/** + * Largest amount of data one media packet can carry over this c2c connection, + * i.e. what has to fit into one datagram once a media connection is up. + */ +unsigned int PICA_media_max_payload(const struct PICA_c2c *c2c); + +/** + * Sends one already formatted protocol message over the media connection. + * The message is dropped, and PICA_OK returned, if the send would block - + * for real time media that is the wanted behaviour. + */ +int PICA_media_send(struct PICA_c2c *c2c, const unsigned char *buf, unsigned int len); + +void PICA_media_fdset(struct PICA_c2n *c2n, fd_set *rfds, int *nfds); +void PICA_media_process(struct PICA_c2n *c2n, fd_set *rfds); + +/* implemented in PICA_client.c, used by PICA_media.c */ + +int PICA_verify_callback(int preverify_ok, X509_STORE_CTX *ctx); +int PICA_verify_peer_cert_common(X509 **peer_cert, SSL *ssl, const unsigned char *peer_id); +unsigned int PICA_media_process_messages(unsigned char *buf, unsigned int *read_pos, struct PICA_c2c *c2c); +void PICA_media_transport_changed(struct PICA_c2c *c2c, int transport, const char *ciphersuitename); +int PICA_media_send_ping(struct PICA_c2c *c2c, int reply); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/PICA_netconf.c b/PICA_netconf.c index 03c12ff..d319633 100644 --- a/PICA_netconf.c +++ b/PICA_netconf.c @@ -173,6 +173,16 @@ int PICA_upnp_autoconfigure_ipv4(int public_port, int local_port, char *public_i else ret = 1; + /* The port with the same number carries call media over UDP, see + * PICA_media.h. Failing to map it only costs calls their UDP transport, + * so it does not change the result reported for the TCP mapping. + */ + if (UPNPCOMMAND_SUCCESS != UPNP_AddPortMapping(urls.controlURL, data.first.servicetype, + ppstr, lpstr, lanaddr, "Pica Pica media", "UDP", 0, "0")) + { + fprintf(stderr, "failed to forward UDP port %s for call media\n", ppstr); + } + freeUPNPDevlist(devlist); FreeUPNPUrls(&urls); return ret; diff --git a/PICA_node.c b/PICA_node.c index 3f61fd0..12a0383 100644 --- a/PICA_node.c +++ b/PICA_node.c @@ -2671,6 +2671,10 @@ void process_listen_read(evutil_socket_t listener, short event, void *arg) nc->state = PICA_NEWCONN_WAITANONTLSACCEPT; IOCTLSETNONBLOCKINGSOCKET(s, 1); + /* the node relays c2c data, so Nagle delays here are added to + * the delay of every message between the two clients + */ + SETTCPNODELAY(s); PICA_debug1("accepted connection from %.16s, new socket %i", inet_ntoa(addr.sin_addr), s);//IPv6 diff --git a/PICA_node.h b/PICA_node.h index 51fc51c..2dbb088 100644 --- a/PICA_node.h +++ b/PICA_node.h @@ -27,6 +27,7 @@ #include <sys/types.h> #include <sys/socket.h> #include <netinet/in.h> +#include <netinet/tcp.h> #include <arpa/inet.h> #include <netdb.h> #include <sys/select.h> diff --git a/doc/proto-doc-latest.txt b/doc/proto-doc-latest.txt index 2ba4cd5..8eab376 100644 --- a/doc/proto-doc-latest.txt +++ b/doc/proto-doc-latest.txt @@ -12,7 +12,7 @@ UH - user hash - the SHA224 hash of a user certificate in DER format, 28 octets. It is used as a user identifier. Connections. -There are 4 types of connections between Pica Pica nodes and clients. +There are 5 types of connections between Pica Pica nodes and clients. -client to node (c2n) - a control connection from client to node @@ -25,7 +25,28 @@ acting as a proxy using the IPv4/IPv6/DNS address of the accepting client previously communicated over a c2c connection. -These connections use TCP transport. The first 3 connection types start +-media client to client (mediac2c) - a direct connection between clients +carrying the real time media of an audio/video call. It uses UDP transport +and DTLS 1.2 with mutual certificate authentication. + +A mediac2c connection is established only while a call is in progress and +only between clients that already have an established directc2c connection. +It uses the address and port number the directc2c connection was made to, +with UDP instead of TCP: the client that made the outgoing directc2c +connection acts as the DTLS client and connects to that same address, the +client that accepted it acts as the DTLS server and listens on the UDP port +with the same number as its directc2c listening port. Each side verifies +that the certificate presented by the peer hashes to the UH of the peer of +the c2c connection the call belongs to, exactly as it does on a directc2c +connection. No addresses are announced separately for this connection type +and no additional protocol messages are needed to establish it. + +Media packets are carried over the mediac2c connection whenever it is +established, and over the c2c or directc2c connection otherwise. There is +no message announcing the change: the receiving side accepts media packets +over either transport for the whole duration of a call. + +The first 4 connection types use TCP transport. The first 3 connection types start with a TLS 1.2 handshake using an anonymous Diffie-Hellman ciphersuite; the connecting host acts as the TLS client, the accepting host as the TLS server. After the connection type is identified by the accepting node, the anonymous @@ -237,6 +258,9 @@ If the client is behind NAT and forwards an open port to its local address, the external port number must be specified in the address. Having received the announcement, the other client can create a connection to that address and close the connection through the node. +The announced port numbers are used both for TCP directc2c connections and for +UDP mediac2c connections, so a client behind NAT has to forward the port for +both protocols. ==================================================== Direct connection attempts failed. @@ -309,7 +333,16 @@ Commands: Audio and Video Calls. -Sent between clients over a c2c or directc2c connection. +The call control messages (0x70 - 0x75) are sent between clients over a c2c or +directc2c connection. +The media packets (0x76, 0x77) are sent over a mediac2c connection when one is +established, and over the c2c or directc2c connection otherwise. Over a +mediac2c connection every datagram contains exactly one message, and messages +can be lost, reordered or duplicated; the receiving side must be prepared for +that and must not treat a malformed or unexpected packet as a protocol error +breaking the connection. In particular, a media packet can arrive before the +0x74 or 0x75 message describing the stream it belongs to, because those travel +over a different connection; such a packet is discarded. ==================================================== Call request (ring) @@ -366,11 +399,9 @@ length - covers everything following it: the 2-octet sequence number, the 4-octet timestamp and the data, i.e. data length + 6. sequence number - incremented by one for every audio packet sent over this -call, wraps around at 65536. Since c2c/directc2c connections are carried -over TCP, packets already arrive in order and without loss; the sequence -number's purpose is to let the receiving side detect encoder-side gaps (e.g. -a dropped capture buffer) and, in the future, to support delivery over an -unordered/lossy transport (e.g. UDP). +call, wraps around at 65536. It lets the receiving side detect encoder-side +gaps (e.g. a dropped capture buffer) and, over a mediac2c connection, put +the received packets back in order, detect losses and discard duplicates. timestamp - the capture timestamp of the packet's first sample, in milliseconds, relative to the moment the call became active (the 0x71 @@ -409,15 +440,36 @@ fragment marker bit (0x8000) set in its sequence number; every preceding fragment of the same frame has that bit cleared. A frame that fits into a single message is sent as one message with the marker bit set. -The receiving side concatenates the data of the received fragments in the -order they arrive, and passes the result to the video decoder once a message -with the marker bit set has been received. +Over a c2c or directc2c connection a single 0x77 message can carry up to +65526 octets of data. Over a mediac2c connection the whole message has to +fit into one datagram, so the amount of data a fragment carries is limited +by the path MTU instead, and an encoded frame is normally split into many +more fragments. + +The receiving side concatenates the data of the received fragments and +passes the result to the video decoder once a message with the marker bit +set has been received. + +Since fragments can be lost over a mediac2c connection, the receiving side +must also check that the fragments it concatenates are consecutive: a +fragment may only start a new frame if the previously accepted message had +the last fragment marker bit set and the counter part of the sequence +number of the new one follows it without a gap. If a gap is detected, the +partially assembled frame is discarded and the receiving side skips +fragments until a message with the last fragment marker bit set is +received, resuming assembly with the message following it. Without this +rule, losing the first fragment of a frame would make the receiving side +treat the second one as the beginning of a frame and pass a truncated frame +to the decoder. ==================================================== Ping. Can be sent by either side, both over the c2n control connection and over a connection between clients. +Over a mediac2c connection it is also used as a keepalive, to hold open the +NAT and firewall state the connection depends on while neither side is +sending media, and to detect that the connection has stopped working. ==================================================== Ping request. diff --git a/pica-client/audiodevice.cpp b/pica-client/audiodevice.cpp index aa4c42a..28f5d0a 100644 --- a/pica-client/audiodevice.cpp +++ b/pica-client/audiodevice.cpp @@ -110,7 +110,8 @@ static SwrContext *makeMonoResampler(AVSampleFormat inFmt, int inRate, AVSampleF } AudioDevice::AudioDevice(QObject *parent) - : QObject(parent), m_sampleRate(48000), m_abort(0) + : QObject(parent), m_sampleRate(48000), m_abort(0), + m_lastPlayedSeq(0), m_havePlayedSeq(false) { } @@ -133,14 +134,48 @@ void AudioDevice::configurePlayback(QString deviceName, QString codec, int sampl m_sampleRate = sampleRate; } -void AudioDevice::enqueuePacket(QByteArray data) +// Difference between two sequence numbers, correct across the point where +// they wrap around: positive if a comes after b. +static inline int seq_diff(quint16 a, quint16 b) +{ + return (qint16)(a - b); +} + +void AudioDevice::enqueuePacket(quint16 seq_num, QByteArray data) { QMutexLocker locker(&m_queueMutex); + int i; + + // Too late to be of any use - the audio it belongs to has already been + // played. This also throws away duplicates of packets already played. + if (m_havePlayedSeq && seq_diff(seq_num, m_lastPlayedSeq) <= 0) + return; + + // Insertion point, and duplicates of packets still waiting. + for (i = m_playQueue.size(); i > 0; i--) + { + int d = seq_diff(seq_num, m_playQueue[i - 1].seq); + + if (d == 0) + return; + + if (d > 0) + break; + } while (m_playQueue.size() >= kMaxQueueDepth) - m_playQueue.dequeue(); + { + m_playQueue.removeFirst(); + + if (i > 0) + i--; + else + return; // the packet just dropped was this one's place + } + + PlaybackPacket p = { seq_num, data }; + m_playQueue.insert(i, p); - m_playQueue.enqueue(data); m_queueCond.wakeAll(); } @@ -151,6 +186,8 @@ void AudioDevice::Close() { QMutexLocker locker(&m_queueMutex); m_playQueue.clear(); + /* the next call starts its own numbering */ + m_havePlayedSeq = false; } m_queueCond.wakeAll(); } @@ -502,7 +539,11 @@ void AudioDevice::Play() if (m_playQueue.isEmpty()) continue; - pktData = m_playQueue.dequeue(); + PlaybackPacket p = m_playQueue.takeFirst(); + + pktData = p.data; + m_lastPlayedSeq = p.seq; + m_havePlayedSeq = true; } AVPacket *in_pkt = av_packet_alloc(); diff --git a/pica-client/audiodevice.h b/pica-client/audiodevice.h index 60f64a5..216657a 100644 --- a/pica-client/audiodevice.h +++ b/pica-client/audiodevice.h @@ -49,7 +49,13 @@ public: // Push a codec packet received from the network into the playback // jitter buffer. Safe to call from any thread. Drops the oldest queued // packet if the buffer is full, favoring low latency over completeness. - Q_INVOKABLE void enqueuePacket(QByteArray data); + // + // seq_num is the packet's 0x76 sequence number. Over a mediac2c + // connection packets can arrive out of order or twice, so the buffer is + // ordered by it, and a packet that arrives after one further along the + // stream has already been played is dropped rather than played out of + // place. + Q_INVOKABLE void enqueuePacket(quint16 seq_num, QByteArray data); public slots: // Blocking capture+encode loop: opens the configured local input @@ -96,7 +102,20 @@ private: // incoming network packet) pushes into it. QMutex m_queueMutex; QWaitCondition m_queueCond; - QQueue<QByteArray> m_playQueue; + + struct PlaybackPacket + { + quint16 seq; + QByteArray data; + }; + + // Kept ordered by sequence number, oldest first. + QList<PlaybackPacket> m_playQueue; + // Sequence number of the last packet handed to the decoder, once one has + // been; anything not newer than it has missed its turn. + quint16 m_lastPlayedSeq; + bool m_havePlayedSeq; + static const int kMaxQueueDepth = 8; // Number of packets Play() waits to have buffered before writing the diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp index 7d3875c..4e91507 100644 --- a/pica-client/audiovideocallcontroller.cpp +++ b/pica-client/audiovideocallcontroller.cpp @@ -45,6 +45,8 @@ AudioVideoCallController::AudioVideoCallController(QObject *parent) connect(skynet, SIGNAL(IncomingVideoParams(QByteArray,QString,quint16,quint16)), this, SLOT(incoming_video_params(QByteArray,QString,quint16,quint16))); connect(skynet, SIGNAL(IncomingVideoPacket(QByteArray,quint16,quint32,QByteArray)), this, SLOT(incoming_video_packet(QByteArray,quint16,quint32,QByteArray))); + connect(skynet, SIGNAL(CallMediaTransportChanged(QByteArray,bool,QString,quint32)), this, SLOT(media_transport_changed(QByteArray,bool,QString,quint32))); + // Earpiece tones (ringback/busy/unreachable) and the incoming call bell // are played by two TonePlayer instances, each living on its own // QThread since TonePlayer::play() blocks the calling thread for the @@ -209,6 +211,13 @@ void AudioVideoCallController::startVideoPipeline() m_videoFrameTimestamp = 0; m_videoFrameStarted = false; + // Datagram sized from the start, even though the call begins on the c2c + // connection where a 0x77 message could hold far more: a media connection + // may come up at any moment during the call, and frames whose fragments + // were cut before that would not fit into a datagram. Over TCP this only + // costs a few more message headers. + cam->setMaxFragmentSize(PICA_MEDIA_SAFE_PAYLOAD); + bool preferCompressed = st.loadValue("video.prefer_compressed", 0).toBool(); bool vaapiEncoding = st.loadValue("video.vaapi_encoding", 0).toBool(); @@ -408,7 +417,6 @@ void AudioVideoCallController::incoming_audio_params(QByteArray peer_id, QString void AudioVideoCallController::incoming_audio_packet(QByteArray peer_id, quint16 seq_num, quint32 timestamp, QByteArray data) { - Q_UNUSED(seq_num); Q_UNUSED(timestamp); if (!is_active || peer_id != m_peer_id) @@ -419,7 +427,7 @@ void AudioVideoCallController::incoming_audio_packet(QByteArray peer_id, quint16 // call from any thread; queuing it behind output's own blocking Play() // call would mean it's never delivered, and Play() would sit forever // pulling from an empty jitter buffer. - output->enqueuePacket(data); + output->enqueuePacket(seq_num, data); } void AudioVideoCallController::video_capture_started(QString codec, int width, int height) @@ -460,6 +468,25 @@ void AudioVideoCallController::send_video_packet(QByteArray data, bool is_last_f skynet->SendVideoPacket(m_peer_id, seq, m_videoFrameTimestamp, data); } +void AudioVideoCallController::media_transport_changed(QByteArray peer_id, bool direct_udp, QString ciphersuitename, quint32 max_payload) +{ + if (peer_id != m_peer_id) + return; + + // Fragments stay datagram sized for the whole call (see + // startVideoPipeline()); this only lowers the size further, for when the + // media connection turns out to carry less than was assumed. Direct call, + // not invokeMethod()/QueuedConnection - see the comment in + // stopAudioPipeline(): cam's thread is blocked inside Capture(). + cam->setMaxFragmentSize(qMin((int)max_payload, (int)PICA_MEDIA_SAFE_PAYLOAD)); + + if (callwindow) + callwindow->setMediaTransport(direct_udp, ciphersuitename); + + qDebug() << "call media transport:" << (direct_udp ? "direct UDP/DTLS" : "c2c connection") + << ciphersuitename << "max payload" << max_payload; +} + void AudioVideoCallController::incoming_video_params(QByteArray peer_id, QString codec, quint16 width, quint16 height) { if (!is_active || peer_id != m_peer_id) diff --git a/pica-client/audiovideocallcontroller.h b/pica-client/audiovideocallcontroller.h index 7d97fac..2a9a0fa 100644 --- a/pica-client/audiovideocallcontroller.h +++ b/pica-client/audiovideocallcontroller.h @@ -115,6 +115,8 @@ private slots: void send_video_packet(QByteArray data, bool is_last_fragment); void incoming_video_params(QByteArray peer_id, QString codec, quint16 width, quint16 height); void incoming_video_packet(QByteArray peer_id, quint16 seq_num, quint32 timestamp, QByteArray data); + + void media_transport_changed(QByteArray peer_id, bool direct_udp, QString ciphersuitename, quint32 max_payload); #ifdef HAVE_VAAPI void video_rendering_failed(); #endif diff --git a/pica-client/callwindow.cpp b/pica-client/callwindow.cpp index 4aff7bc..81959e3 100644 --- a/pica-client/callwindow.cpp +++ b/pica-client/callwindow.cpp @@ -35,6 +35,9 @@ CallWindow::CallWindow(QByteArray peer_id, bool incoming) lbTimer = new QLabel(this); lbTimer->setAlignment(Qt::AlignCenter); lbTimer->hide(); + lbTransport = new QLabel(this); + lbTransport->setAlignment(Qt::AlignCenter); + lbTransport->hide(); lbVideo = new QLabel(this); lbVideo->setAlignment(Qt::AlignCenter); lbVideo->setMinimumSize(320, 240); @@ -49,6 +52,7 @@ CallWindow::CallWindow(QByteArray peer_id, bool incoming) callElapsedSeconds = 0; lv->addWidget(lbTimer); + lv->addWidget(lbTransport); lv->addWidget(lbVideo); #ifdef HAVE_VAAPI lv->addWidget(videoGpu->widget()); @@ -90,6 +94,17 @@ void CallWindow::call_ended() { callTimer->stop(); pbHang->setDisabled(true); + lbTransport->hide(); +} + +void CallWindow::setMediaTransport(bool direct_udp, QString ciphersuitename) +{ + if (direct_udp) + lbTransport->setText(tr("🔐 media: direct UDP, %1").arg(ciphersuitename)); + else + lbTransport->setText(tr("media: through the chat connection")); + + lbTransport->show(); } void CallWindow::showRemoteFrame(QImage frame) diff --git a/pica-client/callwindow.h b/pica-client/callwindow.h index b5b1d97..fbdcf60 100644 --- a/pica-client/callwindow.h +++ b/pica-client/callwindow.h @@ -50,6 +50,9 @@ private: QPushButton *pbCall; QPushButton *pbHang; QLabel *lbTimer; + // How the call's media is being carried, and under which ciphersuite. + // Hidden until that is known. + QLabel *lbTransport; // Displays the remote side's video. Stays hidden until the first frame // arrives, so an audio-only call keeps the compact window it had before. QLabel *lbVideo; @@ -67,6 +70,9 @@ private: public slots: void call_started(); void call_ended(); + // direct_udp tells whether the media is being carried over a direct + // mediac2c connection or over the c2c connection the call was set up on. + void setMediaTransport(bool direct_udp, QString ciphersuitename); void showRemoteFrame(QImage frame); #ifdef HAVE_VAAPI void showRemoteHwFrame(AVFramePtr frame); diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp index e074a48..9e3897b 100644 --- a/pica-client/dialogs/settingsdialog.cpp +++ b/pica-client/dialogs/settingsdialog.cpp @@ -511,8 +511,12 @@ void SettingsDialog::audioTestFeedPacket() return; } - // Direct call, for the same reason as in the call controller. - testSpeaker->enqueuePacket(audioTestPackets.at(audioTestPlaybackPos++)); + // Direct call, for the same reason as in the call controller. Nothing + // here loses or reorders packets, so the sequence numbers are simply the + // order they were recorded in. + testSpeaker->enqueuePacket((quint16)audioTestPlaybackPos, + audioTestPackets.at(audioTestPlaybackPos)); + audioTestPlaybackPos++; } void SettingsDialog::audioTestPlaybackFinished() diff --git a/pica-client/pica-client.pro b/pica-client/pica-client.pro index 87dc462..43111d2 100644 --- a/pica-client/pica-client.pro +++ b/pica-client/pica-client.pro @@ -18,6 +18,7 @@ SOURCES += main.cpp\ callwindow.cpp \ contacts.cpp \ ../PICA_client.c \ + ../PICA_media.c \ ../PICA_msgproc.c \ contactlistwidget.cpp \ mediadevice.cpp \ @@ -56,6 +57,7 @@ HEADERS += mainwindow.h \ contacts.h \ globals.h \ ../PICA_client.h \ + ../PICA_media.h \ ../PICA_msgproc.h \ contactlistwidget.h \ mediadevice.h \ diff --git a/pica-client/skynet.cpp b/pica-client/skynet.cpp index 0ca37cc..7427cd4 100644 --- a/pica-client/skynet.cpp +++ b/pica-client/skynet.cpp @@ -70,7 +70,8 @@ SkyNet::SkyNet() call_audio_params_cb, call_video_params_cb, call_audio_packet_cb, - call_video_packet_cb + call_video_packet_cb, + call_media_transport_cb }; @@ -84,6 +85,8 @@ SkyNet::SkyNet() connect(this, SIGNAL(MultiloginMessageReceived(quint64,QString,quint16)), this, SLOT(multilogin_event(quint64,QString,quint16)), Qt::QueuedConnection); file_transfer_timer_id = 0; + call_timer_id = 0; + call_active = false; } @@ -138,6 +141,43 @@ void SkyNet::active_filetransfers_down() } } +void SkyNet::call_active_set(bool active, QByteArray peer) +{ + if (active) + { + call_active_peer = peer; + } + else + { + // A c2c connection to some other contact going down says nothing + // about the call in progress. + if (!peer.isEmpty() && !call_active_peer.isEmpty() && peer != call_active_peer) + return; + + call_active_peer.clear(); + } + + if (active == call_active) + return; + + call_active = active; + + // Same trick as active_filetransfers_up(): a zero interval timer is not + // a busy loop here, because PICA_event_loop() spends its time blocked + // inside select(). Without it the network thread only enters select() + // once per event_loop_timer_id tick, and an incoming media packet waits + // for the rest of that interval before it is read. + if (active && call_timer_id == 0) + { + call_timer_id = startTimer(0); + } + else if (!active && call_timer_id != 0) + { + killTimer(call_timer_id); + call_timer_id = 0; + } +} + void SkyNet::active_filetransfers_reset() { active_filetransfers = 0; @@ -379,7 +419,8 @@ void SkyNet::reconnect_c2c() void SkyNet::timerEvent(QTimerEvent *e) { - if (e->timerId() == file_transfer_timer_id || e->timerId() == event_loop_timer_id) + if (e->timerId() == file_transfer_timer_id || e->timerId() == event_loop_timer_id + || e->timerId() == call_timer_id) { if (connecting_nodes.size() > 0 || self_aware) { @@ -396,7 +437,12 @@ void SkyNet::timerEvent(QTimerEvent *e) nodelinks.append(NULL); - ret = PICA_event_loop(nodelinks.data(), 25); + // A shorter select() timeout while a call is in progress: the + // audio and video threads hand their packets over with a queued + // invokeMethod(), and Qt cannot deliver those to this thread + // while it sits inside select(). The timeout is therefore the + // worst case delay added to every outgoing media packet. + ret = PICA_event_loop(nodelinks.data(), call_active ? 5 : 25); while(!connected_nodes_to_close.empty()) { @@ -561,6 +607,7 @@ void SkyNet::do_Exit() self_aware = false; active_filetransfers_reset(); + call_active_set(false, QByteArray()); msgqueues.clear(); @@ -812,6 +859,8 @@ void SkyNet::do_AcceptCall(QByteArray from) if (ret != PICA_OK) emit CallFailed(from, QString(tr("Failed to pickup the call: %1")).arg(ret)); + else + call_active_set(true, from); return; } @@ -836,6 +885,8 @@ void SkyNet::do_RejectCall(QByteArray from) if (ret != PICA_OK) emit CallFailed(from, QString(tr("Failed to reject the call: %1")).arg(ret)); + call_active_set(false, from); + return; } @@ -859,6 +910,8 @@ void SkyNet::do_HangupCall(QByteArray with) if (ret != PICA_OK) emit CallFailed(with, QString(tr("Failed to hang up the call: %1")).arg(ret)); + call_active_set(false, with); + return; } @@ -1229,6 +1282,11 @@ void SkyNet::emit_IncomingVideoPacket(QByteArray peer_id, quint16 seq_num, quint emit IncomingVideoPacket(peer_id, seq_num, timestamp, data); } +void SkyNet::emit_CallMediaTransportChanged(QByteArray peer_id, bool direct_udp, QString ciphersuitename, quint32 max_payload) +{ + emit CallMediaTransportChanged(peer_id, direct_udp, ciphersuitename, max_payload); +} + //callbacks void SkyNet::newmsg_cb(const unsigned char *peer_id, const char *msgbuf, unsigned int nb, int type) @@ -1297,6 +1355,8 @@ void SkyNet::c2c_closed_cb(const unsigned char *peer_id, int reason) skynet->emit_c2cClosed(QByteArray((const char*)peer_id, PICA_ID_SIZE)); skynet->emit_ConnectionStatusUpdated(QByteArray((const char*)peer_id, PICA_ID_SIZE), QString(tr("Disconnected"))); + /* a call carried by this connection cannot continue */ + skynet->call_active_set(false, QByteArray((const char*)peer_id, PICA_ID_SIZE)); } void SkyNet::nodelist_cb(int type, void *addr_bin, const char *addr_str, unsigned int port) @@ -1450,17 +1510,26 @@ void SkyNet::incoming_call_cb(const unsigned char *peer_id) void SkyNet::call_picked_up_cb(const unsigned char *peer_id) { - skynet->emit_CallAccepted(QByteArray((const char*)peer_id, PICA_ID_SIZE)); + QByteArray peer((const char*)peer_id, PICA_ID_SIZE); + + skynet->call_active_set(true, peer); + skynet->emit_CallAccepted(peer); } void SkyNet::call_rejected_cb(const unsigned char *peer_id) { - skynet->emit_CallRejected(QByteArray((const char*)peer_id, PICA_ID_SIZE)); + QByteArray peer((const char*)peer_id, PICA_ID_SIZE); + + skynet->call_active_set(false, peer); + skynet->emit_CallRejected(peer); } void SkyNet::call_hangup_cb(const unsigned char *peer_id) { - skynet->emit_CallHungup(QByteArray((const char*)peer_id, PICA_ID_SIZE)); + QByteArray peer((const char*)peer_id, PICA_ID_SIZE); + + skynet->call_active_set(false, peer); + skynet->emit_CallHungup(peer); } void SkyNet::call_audio_params_cb(const unsigned char *peer_id, const char *codec, uint16_t sample_rate) @@ -1482,3 +1551,17 @@ void SkyNet::call_video_packet_cb(const unsigned char *peer_id, uint16_t seq_num { skynet->emit_IncomingVideoPacket(QByteArray((const char*)peer_id, PICA_ID_SIZE), seq_num, timestamp, QByteArray(pkt_data, size)); } + +void SkyNet::call_media_transport_cb(const unsigned char *peer_id, int transport, const char *ciphersuitename, unsigned int max_payload) +{ + QByteArray peer((const char*)peer_id, PICA_ID_SIZE); + bool direct_udp = (transport == PICA_CALL_TRANSPORT_MEDIAC2C); + + skynet->emit_ConnectionStatusUpdated(peer, direct_udp + ? QString("🔐: %1 media").arg(ciphersuitename) + : QString(tr("call media over the c2c connection"))); + + skynet->emit_CallMediaTransportChanged(peer, direct_udp, + ciphersuitename ? QString::fromLatin1(ciphersuitename) : QString(), + max_payload); +} diff --git a/pica-client/skynet.h b/pica-client/skynet.h index 0baf496..8b46507 100644 --- a/pica-client/skynet.h +++ b/pica-client/skynet.h @@ -19,6 +19,7 @@ #include "nodes.h" #include "../PICA_client.h" +#include "../PICA_media.h" #include "../PICA_proto.h" #include "accounts.h" #include <QObject> @@ -95,6 +96,12 @@ signals: void IncomingVideoParams(QByteArray peer_id, QString codec, quint16 width, quint16 height); void IncomingVideoPacket(QByteArray peer_id, quint16 seq_num, quint32 timestamp, QByteArray data); + + /** The media packets of the call started or stopped being carried over a + * direct UDP (mediac2c) connection. max_payload is the largest amount of + * data one media packet can carry from now on. + */ + void CallMediaTransportChanged(QByteArray peer_id, bool direct_udp, QString ciphersuitename, quint32 max_payload); private: // Constructed in init(), once this SkyNet instance is running on its // own network thread and that thread's dedicated QSqlDatabase @@ -118,6 +125,7 @@ private: int event_loop_timer_id; int file_transfer_timer_id; int c2c_reconnect_timer_id; + int call_timer_id; void timerEvent(QTimerEvent * e); @@ -133,6 +141,13 @@ private: void active_filetransfers_reset(); quint32 active_filetransfers; + /* Only one call can be in progress at a time, so this is a flag and not + * a counter like active_filetransfers. + */ + void call_active_set(bool active, QByteArray peer); + bool call_active; + QByteArray call_active_peer; + void continue_start_call(); // check if there is a call waiting to be initiated with peer_id bool is_call_waiting_for(const QByteArray &peer_id); @@ -174,6 +189,8 @@ private: void emit_IncomingVideoParams(QByteArray peer_id, QString codec, quint16 width, quint16 height); void emit_IncomingVideoPacket(QByteArray peer_id, quint16 seq_num, quint32 timestamp, QByteArray data); + void emit_CallMediaTransportChanged(QByteArray peer_id, bool direct_udp, QString ciphersuitename, quint32 max_payload); + static void newmsg_cb(const unsigned char *peer_id, const char* msgbuf, unsigned int nb, int type); static void msgok_cb(const unsigned char *peer_id); @@ -232,6 +249,8 @@ private: static void call_video_packet_cb(const unsigned char *peer_id, uint16_t seq_num, uint32_t timestamp, uint16_t size, const char *pkt_data); + static void call_media_transport_cb(const unsigned char *peer_id, int transport, const char *ciphersuitename, unsigned int max_payload); + private slots: void nodelink_activated(PICA_c2n *c2n); void nodelink_closed(PICA_c2n *c2n, int error); diff --git a/pica-client/videodevice.cpp b/pica-client/videodevice.cpp index 4fd03d6..52367af 100644 --- a/pica-client/videodevice.cpp +++ b/pica-client/videodevice.cpp @@ -16,6 +16,7 @@ */ #include "videodevice.h" #include "../PICA_proto.h" +#include "../PICA_media.h" #include <QFile> #include <QDebug> @@ -62,11 +63,18 @@ static const int64_t kBitrate = 600000; // can resynchronize. static const int kGopSize = kFrameRate * 2; -// Largest amount of encoded data one 0x77 message can carry: the protocol's -// per-message payload cap, minus the sequence number + timestamp header that -// every call media packet carries. +// Largest amount of encoded data one 0x77 message can carry over a c2c or +// directc2c connection: the protocol's per-message payload cap, minus the +// sequence number + timestamp header that every call media packet carries. +// Over a mediac2c connection the limit is the path MTU instead and is set +// with setMaxFragmentSize() once that connection is up. static const int kMaxFragmentSize = PICA_PROTO_C2CMSG_MAXDATASIZE - PICA_PROTO_CALL_PACKET_HDRSIZE; +// Largest encoded slice the software encoder is asked to produce: one that +// still fits into a single datagram of a mediac2c connection, so that losing +// a datagram costs a slice of the picture rather than a whole frame. +static const int kSliceMaxSize = PICA_MEDIA_SAFE_PAYLOAD; + static QString ff_errstr(int err) { char buf[256] = {0}; @@ -153,7 +161,8 @@ QStringList VideoDevice::CompressedFormats(const QString &device) } VideoFrameAssembler::VideoFrameAssembler() - : m_timestamp(0), m_inProgress(false) + : m_timestamp(0), m_inProgress(false), m_lastSeq(0), + m_lastWasFrameEnd(false), m_haveLastSeq(false) { } @@ -161,10 +170,49 @@ void VideoFrameAssembler::reset() { m_buffer.clear(); m_inProgress = false; + m_haveLastSeq = false; + // The first fragment received is as likely to be the middle of a frame as + // its beginning, so assembly only starts after the first frame boundary. + m_lastWasFrameEnd = false; } QByteArray VideoFrameAssembler::addFragment(quint16 seq_num, quint32 timestamp, const QByteArray &data) { + const quint16 counter = seq_num & SeqNumMask; + const bool isLast = (seq_num & LastFragmentFlag) != 0; + bool contiguous = true; + + // Over a mediac2c connection fragments can be lost, so a fragment is only + // accepted if it directly follows the previous one. When it does not, + // what is in the buffer is incomplete and everything up to the end of the + // current frame has to be skipped: a fragment may only begin a frame if + // the one before it ended a frame. Without that rule, losing a frame's + // first fragment would make its second fragment look like the beginning + // of a frame and hand the decoder a truncated one. + if (m_haveLastSeq) + contiguous = (counter == ((m_lastSeq + 1) & SeqNumMask)); + + m_lastSeq = counter; + m_haveLastSeq = true; + + if (!contiguous) + { + // Resynchronize at the next frame boundary. This fragment can still + // start a frame itself, if the gap swallowed whole frames and it + // happens to be a first fragment - which is what a matching new + // timestamp on an empty buffer means. + m_buffer.clear(); + m_inProgress = false; + m_lastWasFrameEnd = false; + } + + if (!m_lastWasFrameEnd && !m_inProgress) + { + // Still in the middle of a frame whose beginning was lost. + m_lastWasFrameEnd = isLast; + return QByteArray(); + } + // Fragments of one encoded frame all carry the same timestamp; a // different one while a frame is still being assembled means the rest of // that frame is never coming, so drop what was collected and start over. @@ -174,20 +222,32 @@ QByteArray VideoFrameAssembler::addFragment(quint16 seq_num, quint32 timestamp, m_buffer.append(data); m_timestamp = timestamp; m_inProgress = true; + m_lastWasFrameEnd = isLast; - if (!(seq_num & LastFragmentFlag)) + if (!isLast) return QByteArray(); QByteArray frame = m_buffer; - reset(); + + m_buffer.clear(); + m_inProgress = false; return frame; } VideoDevice::VideoDevice(QObject *parent) : QObject(parent), m_width(640), m_height(480), m_preferCompressed(false), - m_useVaapi(false), m_useVaapiRender(false), m_abort(0) + m_useVaapi(false), m_useVaapiRender(false), m_abort(0), + m_maxFragmentSize(kMaxFragmentSize) +{ +} + +void VideoDevice::setMaxFragmentSize(int size) { + if (size <= 0 || size > kMaxFragmentSize) + size = kMaxFragmentSize; + + m_maxFragmentSize.storeRelaxed(size); } VideoDevice::~VideoDevice() @@ -342,10 +402,14 @@ void VideoDevice::emitFragments(const unsigned char *data, int size) // is flagged so the receiver knows the frame is complete (see the // fragmentation notes in the protocol doc). int offset = 0; + // Read once per frame rather than per fragment, so that a transport + // change in the middle of a frame cannot make the receiving side see a + // frame whose fragments were sized by two different rules. + const int maxFragment = m_maxFragmentSize.loadRelaxed(); while (offset < size) { - int chunk = qMin(kMaxFragmentSize, size - offset); + int chunk = qMin(maxFragment, size - offset); bool isLast = (offset + chunk >= size); emit packetReady(QByteArray((const char *)data + offset, chunk), isLast); @@ -520,6 +584,13 @@ void VideoDevice::runTranscodeLoop(AVFormatContext *ifmt_ctx, AVStream *in_st) av_opt_set(enc_ctx->priv_data, "preset", "veryfast", 0); av_opt_set(enc_ctx->priv_data, "tune", "zerolatency", 0); + // Keeps every slice within one datagram of a mediac2c connection, so + // that a lost fragment costs a slice of the picture rather than the + // whole frame. Harmless when the media goes over TCP instead - it + // only splits frames into more NAL units than strictly necessary. + av_opt_set(enc_ctx->priv_data, "x264opts", + QString("slice-max-size=%1").arg(kSliceMaxSize).toUtf8().constData(), 0); + if ((ret = avcodec_open2(enc_ctx, enc, nullptr)) < 0) { QString msg = QString("Could not open H.264 encoder: %1").arg(ff_errstr(ret)); diff --git a/pica-client/videodevice.h b/pica-client/videodevice.h index 320d55a..58eaef9 100644 --- a/pica-client/videodevice.h +++ b/pica-client/videodevice.h @@ -44,7 +44,8 @@ public: // Adds one received fragment. Returns the complete encoded frame once // the fragment carrying the last fragment marker has been added, and an - // empty QByteArray while the frame is still incomplete. + // empty QByteArray while the frame is still incomplete or while waiting + // to resynchronize after a loss. QByteArray addFragment(quint16 seq_num, quint32 timestamp, const QByteArray &data); void reset(); @@ -53,6 +54,13 @@ private: QByteArray m_buffer; quint32 m_timestamp; bool m_inProgress; + // Counter part of the sequence number of the last accepted fragment, and + // whether that fragment ended a frame. Together they say whether the + // fragment being added continues what came before it or whether + // something was lost on the way - see addFragment(). + quint16 m_lastSeq; + bool m_lastWasFrameEnd; + bool m_haveLastSeq; }; // Video counterpart of AudioDevice: a single instance is dedicated to one @@ -102,6 +110,14 @@ public: // video, showing the newest frame matters more than showing every frame. void enqueueFrame(QByteArray encodedFrame); + // Largest amount of encoded data one fragment emitted by packetReady() + // may carry. It changes while a call is running: a media packet sent over + // a mediac2c connection has to fit into a single datagram, which is far + // less than a 0x77 message can hold over TCP. Safe to call from any + // thread, and called directly rather than queued for the same reason + // enqueueFrame() is - the capture loop it affects is already running. + void setMaxFragmentSize(int size); + // Stops keeping decoded frames in GPU memory, for when the renderer that // was to draw them turns out not to be able to: decoding carries on, but // frames come back through system memory and are emitted by frameReady() @@ -183,6 +199,9 @@ private: // loop is running, so it has to be read atomically rather than as a plain // bool like the settings above, which are fixed before the loop starts. QAtomicInt m_hwRenderDisabled; + // Likewise set from another thread, by setMaxFragmentSize(), while the + // capture loop is running. + QAtomicInt m_maxFragmentSize; QMutex m_queueMutex; QWaitCondition m_queueCond; |