Start of "Simple Peer-To-Peer Security" protocol.

Encryption and authentication of the meta connection is spread out over
meta.c and protocol_auth.c. The new protocol was added there as well,
leading to spaghetti code. To improve things, the new protocol will now
be implemented in sptps.[ch].

The goal is to have a very simplified version of TLS. There is a record
layer, and there are only two record types: application data and
handshake messages. The handshake message contains a random nonce, an
ephemeral ECDH public key, and an ECDSA signature over the former. After
the ECDH public keys are exchanged, a shared secret is calculated, and a
TLS style PRF is used to generate the key material for the cipher and
HMAC algorithm, and further communication is encrypted and authenticated.

A lot of the simplicity comes from the fact that both sides must have
each other's public keys in advance, and there are no options to choose.
There will be one fixed cipher suite, and both peers always authenticate
each other. (Inspiration taken from Ian Grigg's hypotheses[0].)
There might be some compromise in the future, to enable or disable
encryption, authentication and compression, but there will be no choice
of algorithms. This will allow SPTPS to be built with a few embedded
crypto algorithms instead of linking with huge crypto libraries.

The API is also kept simple. There is a start and a stop function. All
data necessary to make the connection work is passed in the start
function. Instead having both send- and receive-record functions, there
is a send-record function and a receive-data function. The latter will
pass protocol data received from the peer to the SPTPS implementation,
which will in turn call a receive-record callback function when
necessary. This hides all the handshaking from the application, and is
completely independent from any event loop or socket characteristics.

[0] http://iang.org/ssl/hn_hypotheses_in_secure_protocol_design.html

src/openssl/cipher.c

@@ -101,13 +101,13 @@ bool cipher_encrypt(cipher_t *cipher, const void *indata, size_t inlen, void *ou
 		if(EVP_EncryptInit_ex(&cipher->ctx, NULL, NULL, NULL, NULL)
 				&& EVP_EncryptUpdate(&cipher->ctx, (unsigned char *)outdata, &len, indata, inlen)
 				&& EVP_EncryptFinal(&cipher->ctx, (unsigned char *)outdata + len, &pad)) {
-			*outlen = len + pad;
+			if(outlen) *outlen = len + pad;
 			return true;
 		}
 	} else {
 		int len;
 		if(EVP_EncryptUpdate(&cipher->ctx, outdata, &len, indata, inlen)) {
-			*outlen = len;
+			if(outlen) *outlen = len;
 			return true;
 		}
 	}
@@ -122,13 +122,13 @@ bool cipher_decrypt(cipher_t *cipher, const void *indata, size_t inlen, void *ou
 		if(EVP_DecryptInit_ex(&cipher->ctx, NULL, NULL, NULL, NULL)
 				&& EVP_DecryptUpdate(&cipher->ctx, (unsigned char *)outdata, &len, indata, inlen)
 				&& EVP_DecryptFinal(&cipher->ctx, (unsigned char *)outdata + len, &pad)) {
-			*outlen = len + pad;
+			if(outlen) *outlen = len + pad;
 			return true;
 		}
 	} else {
 		int len;
 		if(EVP_EncryptUpdate(&cipher->ctx, outdata, &len, indata, inlen)) {
-			*outlen = len;
+			if(outlen) *outlen = len;
 			return true;
 		}
 	}

src/openssl/digest.c

@@ -115,6 +115,10 @@ int digest_get_nid(const digest_t *digest) {
 	return digest->digest ? digest->digest->type : 0;
 }
 
+size_t digest_keylength(const digest_t *digest) {
+	return digest->digest->md_size;
+}
+
 size_t digest_length(const digest_t *digest) {
 	return digest->maclength;
 }

src/openssl/digest.h

@@ -39,6 +39,7 @@ extern bool digest_create(struct digest *, const void *indata, size_t inlen, voi
 extern bool digest_verify(struct digest *, const void *indata, size_t inlen, const void *digestdata);
 extern bool digest_set_key(struct digest *, const void *key, size_t len);
 extern int digest_get_nid(const struct digest *);
+extern size_t digest_keylength(const struct digest *);
 extern size_t digest_length(const struct digest *);
 extern bool digest_active(const struct digest *);
 

src/openssl/prf.c

@@ -26,7 +26,7 @@
    We use SHA512 and Whirlpool instead of MD5 and SHA1.
  */
 
-static bool prf_xor(int nid, char *secret, size_t secretlen, char *seed, size_t seedlen, char *out, ssize_t outlen) {
+static bool prf_xor(int nid, const char *secret, size_t secretlen, char *seed, size_t seedlen, char *out, ssize_t outlen) {
 	digest_t digest;
 	
 	if(!digest_open_by_nid(&digest, nid, -1))
@@ -65,7 +65,7 @@ static bool prf_xor(int nid, char *secret, size_t secretlen, char *seed, size_t
 	return true;
 }
 
-bool prf(char *secret, size_t secretlen, char *seed, size_t seedlen, char *out, size_t outlen) {
+bool prf(const char *secret, size_t secretlen, char *seed, size_t seedlen, char *out, size_t outlen) {
 	/* Split secret in half, generate outlen bits with two different hash algorithms,
 	   and XOR the results. */
 

src/openssl/prf.h

@@ -20,6 +20,6 @@
 #ifndef __TINC_PRF_H__
 #define __TINC_PRF_H__
 
-extern bool prf(char *secret, size_t secretlen, char *seed, size_t seedlen, char *out, size_t outlen);
+extern bool prf(const char *secret, size_t secretlen, char *seed, size_t seedlen, char *out, size_t outlen);
 
 #endif