changeset 12684:ca461a33aa8b

8130875: Ucrypto library leaks memory when null output buffer is specified Summary: Avoid null output buffer to work around Solaris memory leak bug in Ucrypto library Reviewed-by: ascarpino
author valeriep
date Fri, 04 Sep 2015 19:55:40 +0000
parents a08b835d32ba
children 4226ee94c70d
files src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c
diffstat 1 files changed, 11 insertions(+), 3 deletions(-) [+]
line wrap: on
line diff
--- a/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c	Fri Sep 04 12:18:03 2015 +0200
+++ b/src/jdk.crypto.ucrypto/solaris/native/libj2ucrypto/nativeCrypto.c	Fri Sep 04 19:55:40 2015 +0000
@@ -435,6 +435,11 @@
   int rv = 0;
 
   context = (crypto_ctx_t *) pContext;
+  // Avoid null output buffer to workaround Solaris bug21481818 (fixed in S12)
+  if (bufOut == NULL) {
+    bufOut = (unsigned char *)(&outLen);
+    outLen = 0;
+  }
   rv = CipherFinal(context, encrypt, (unsigned char*)bufOut, outOfs, &outLen);
   free(context);
   if (rv) {
@@ -648,7 +653,8 @@
 
   // out is null when nativeFinal() is called solely for resource clean up
   if (out == NULL) {
-    bufOut = NULL;
+    // Avoid null output buffer to workaround Solaris bug21481818 (fixed in S12)
+    bufOut = (unsigned char *)(&outLen);
     outLen = 0;
   } else {
     outLen = (*env)->GetArrayLength(env, out) - outOfs;
@@ -661,10 +667,12 @@
   rv = CipherFinal(context, encrypt, bufOut, 0, &outLen);
   if (rv) {
     free(context);
-    free(bufOut);
+    if (outLen != 0) {
+      free(bufOut);
+    }
     return -rv;
   } else {
-    if (bufOut != NULL) {
+    if (bufOut != NULL && outLen != 0) {
       (*env)->SetByteArrayRegion(env, out, outOfs, outLen, (jbyte *)bufOut);
       free(bufOut);
     }