openssh unix dev - Nov 2005 - [PATCH] Solaris 10 and missing OpenSSL functions >128bit

Hi all.

Solaris 10's default libcrypto does not have support for AES 192 and 256
bit functions.  The attached patch, against -current, and based partially
on an earlier one by djm, will use OpenSSH's builtin rijndael code for
all AES crypto functions and thus will allow it to build and function
on Solaris 10 without the extra crypto packages (SUNWcry, SUNWcryr)
or a locally built OpenSSL.

Tested OK for me on Solaris 10/x86 but other testing appreciated (eg SPARC,
AMD64).

Note that it does not use the native OpenSSL's AES128 functions (even
though that would be possible) and thus will not benefit from any
optimizations or hardware support that may be present in it.  This is
deliberate, as using the builtin rijndael for all AES is a tested
configuration whereas only using it for some is not.  This can be changed
later if it is proven safe.

Also note that the regress tests will fail on the SSHv1 Blowfish test
since that uses a 256 bit key, which is also broken in that OpenSSL.
This will cause a runtime failure:

$ ssh -1 -c blowfish localhost
[...]
cipher_init: EVP_CipherInit: set key failed for blowfish

Index: cipher-aes.c
==================================================================RCS file:
/usr/local/src/security/openssh/cvs/openssh_cvs/cipher-aes.c,v
retrieving revision 1.4
diff -u -p -r1.4 cipher-aes.c
--- cipher-aes.c	9 Dec 2003 08:05:43 -0000	1.4
+++ cipher-aes.c	20 Nov 2005 02:39:30 -0000
@@ -23,7 +23,11 @@
  */
 
 #include "includes.h"
-#if OPENSSL_VERSION_NUMBER < 0x00907000L
+
+/* compatibility with old or broken OpenSSL versions */
+#include "openbsd-compat/openssl-compat.h"
+
+#ifdef USE_BUILTIN_RIJNDAEL
 RCSID("$OpenBSD: cipher-aes.c,v 1.2 2003/11/26 21:44:29 djm Exp $");
 
 #include <openssl/evp.h>
@@ -31,10 +35,6 @@ RCSID("$OpenBSD: cipher-aes.c,v 1.2 2003
 #include "xmalloc.h"
 #include "log.h"
 
-#if OPENSSL_VERSION_NUMBER < 0x00906000L
-#define SSH_OLD_EVP
-#endif
-
 #define RIJNDAEL_BLOCKSIZE 16
 struct ssh_rijndael_ctx
 {
@@ -157,4 +157,4 @@ evp_rijndael(void)
 #endif
 	return (&rijndal_cbc);
 }
-#endif /* OPENSSL_VERSION_NUMBER */
+#endif /* USE_BUILTIN_RIJNDAEL */
Index: cipher-ctr.c
==================================================================RCS file:
/usr/local/src/security/openssh/cvs/openssh_cvs/cipher-ctr.c,v
retrieving revision 1.7
diff -u -p -r1.7 cipher-ctr.c
--- cipher-ctr.c	17 Jul 2005 07:22:45 -0000	1.7
+++ cipher-ctr.c	20 Nov 2005 02:37:32 -0000
@@ -21,11 +21,10 @@ RCSID("$OpenBSD: cipher-ctr.c,v 1.6 2005
 #include "log.h"
 #include "xmalloc.h"
 
-#if OPENSSL_VERSION_NUMBER < 0x00906000L
-#define SSH_OLD_EVP
-#endif
+/* compatibility with old or broken OpenSSL versions */
+#include "openbsd-compat/openssl-compat.h"
 
-#if OPENSSL_VERSION_NUMBER < 0x00907000L
+#ifdef USE_BUILTIN_RIJNDAEL
 #include "rijndael.h"
 #define AES_KEY rijndael_ctx
 #define AES_BLOCK_SIZE 16
Index: cipher.c
==================================================================RCS file:
/usr/local/src/security/openssh/cvs/openssh_cvs/cipher.c,v
retrieving revision 1.81
diff -u -p -r1.81 cipher.c
--- cipher.c	17 Jul 2005 07:02:10 -0000	1.81
+++ cipher.c	20 Nov 2005 02:22:41 -0000
@@ -334,7 +334,7 @@ cipher_get_keyiv(CipherContext *cc, u_ch
 		if ((u_int)evplen != len)
 			fatal("%s: wrong iv length %d != %d", __func__,
 			    evplen, len);
-#if OPENSSL_VERSION_NUMBER < 0x00907000L
+#ifdef USE_BUILTIN_RIJNDAEL
 		if (c->evptype == evp_rijndael)
 			ssh_rijndael_iv(&cc->evp, 0, iv, len);
 		else
@@ -365,7 +365,7 @@ cipher_set_keyiv(CipherContext *cc, u_ch
 		evplen = EVP_CIPHER_CTX_iv_length(&cc->evp);
 		if (evplen == 0)
 			return;
-#if OPENSSL_VERSION_NUMBER < 0x00907000L
+#ifdef USE_BUILTIN_RIJNDAEL
 		if (c->evptype == evp_rijndael)
 			ssh_rijndael_iv(&cc->evp, 1, iv, evplen);
 		else
Index: configure.ac
==================================================================RCS file:
/usr/local/src/security/openssh/cvs/openssh_cvs/configure.ac,v
retrieving revision 1.307
diff -u -p -r1.307 configure.ac
--- configure.ac	12 Nov 2005 07:42:37 -0000	1.307
+++ configure.ac	20 Nov 2005 01:20:40 -0000
@@ -1745,6 +1745,24 @@ Also see contrib/findssl.sh for help ide
 	]
 )
 
+# Check for OpenSSL without EVP_aes_{192,256}_cbc
+AC_MSG_CHECKING([whether OpenSSL has crippled AES support])
+AC_COMPILE_IFELSE(
+	[AC_LANG_SOURCE([[
+#include <string.h>
+#include <openssl/evp.h>
+int main(void) { exit(EVP_aes_192_cbc() == NULL || EVP_aes_256_cbc() == NULL)}
+	]])],
+	[
+		AC_MSG_RESULT(no)
+	],
+	[
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(OPENSSL_LOBOTOMISED_AES, 1,
+		    [libcrypto is missing AES 192 and 256 bit functions])
+	]
+)
+
 # Some systems want crypt() from libcrypt, *not* the version in OpenSSL,
 # because the system crypt() is more featureful.
 if test "x$check_for_libcrypt_before" = "x1"; then
Index: openbsd-compat/openssl-compat.h
==================================================================RCS file:
/usr/local/src/security/openssh/cvs/openssh_cvs/openbsd-compat/openssl-compat.h,v
retrieving revision 1.1
diff -u -p -r1.1 openssl-compat.h
--- openbsd-compat/openssl-compat.h	9 Jun 2005 11:45:11 -0000	1.1
+++ openbsd-compat/openssl-compat.h	20 Nov 2005 02:30:01 -0000
@@ -24,7 +24,11 @@
 # define EVP_CIPHER_CTX_get_app_data(e)		((e)->app_data)
 #endif
 
-#if OPENSSL_VERSION_NUMBER < 0x00907000L
+#if (OPENSSL_VERSION_NUMBER < 0x00907000L) ||
defined(OPENSSL_LOBOTOMISED_AES)
+# define USE_BUILTIN_RIJNDAEL
+#endif
+
+#ifdef USE_BUILTIN_RIJNDAEL
 # define EVP_aes_128_cbc evp_rijndael
 # define EVP_aes_192_cbc evp_rijndael
 # define EVP_aes_256_cbc evp_rijndael

-- 
Darren Tucker (dtucker at zip.com.au)
GPG key 8FF4FA69 / D9A3 86E9 7EEE AF4B B2D4  37C9 C982 80C7 8FF4 FA69
    Good judgement comes with experience. Unfortunately, the experience
usually comes from bad judgement.