Properly error out when AEAD Algorithms are used with Encrypted Data

Encrypted Data does not support AEAD algorithms. If you wish to
use AEAD algorithms you will have to use a CMS_AuthEnvelopedData
structure. Therefore, when AEAD algorithms are used with
CMS_EncryptedData_set1_key will now return an error.

Fixes: #28607

Reviewed-by: Paul Dale <paul.dale@oracle.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/28711)

(cherry picked from commit 86344acb02)

crypto/cms/cms_enc.c

@@ -229,6 +229,10 @@ int CMS_EncryptedData_set1_key(CMS_ContentInfo *cms, const EVP_CIPHER *ciph,
         return 0;
     }
     if (ciph) {
+        if ((EVP_CIPHER_get_flags(ciph) & EVP_CIPH_FLAG_AEAD_CIPHER) != 0) {
+            ERR_raise(ERR_LIB_CMS, CMS_R_UNSUPPORTED_CONTENT_ENCRYPTION_ALGORITHM);
+            return 0;
+        }
         if (cms->d.encryptedData != NULL) {
             M_ASN1_free_of(cms->d.encryptedData, CMS_EncryptedData);
             cms->d.encryptedData = NULL;

doc/build.info

@@ -791,6 +791,10 @@ DEPEND[html/man3/CMS_EncryptedData_encrypt.html]=man3/CMS_EncryptedData_encrypt.
 GENERATE[html/man3/CMS_EncryptedData_encrypt.html]=man3/CMS_EncryptedData_encrypt.pod
 DEPEND[man/man3/CMS_EncryptedData_encrypt.3]=man3/CMS_EncryptedData_encrypt.pod
 GENERATE[man/man3/CMS_EncryptedData_encrypt.3]=man3/CMS_EncryptedData_encrypt.pod
+DEPEND[html/man3/CMS_EncryptedData_set1_key.html]=man3/CMS_EncryptedData_set1_key.pod
+GENERATE[html/man3/CMS_EncryptedData_set1_key.html]=man3/CMS_EncryptedData_set1_key.pod
+DEPEND[man/man3/CMS_EncryptedData_set1_key.3]=man3/CMS_EncryptedData_set1_key.pod
+GENERATE[man/man3/CMS_EncryptedData_set1_key.3]=man3/CMS_EncryptedData_set1_key.pod
 DEPEND[html/man3/CMS_EnvelopedData_create.html]=man3/CMS_EnvelopedData_create.pod
 GENERATE[html/man3/CMS_EnvelopedData_create.html]=man3/CMS_EnvelopedData_create.pod
 DEPEND[man/man3/CMS_EnvelopedData_create.3]=man3/CMS_EnvelopedData_create.pod
@@ -3125,6 +3129,7 @@ html/man3/BN_zero.html \
 html/man3/BUF_MEM_new.html \
 html/man3/CMS_EncryptedData_decrypt.html \
 html/man3/CMS_EncryptedData_encrypt.html \
+html/man3/CMS_EncryptedData_set1_key.html \
 html/man3/CMS_EnvelopedData_create.html \
 html/man3/CMS_add0_cert.html \
 html/man3/CMS_add1_recipient_cert.html \
@@ -3769,6 +3774,7 @@ man/man3/BN_zero.3 \
 man/man3/BUF_MEM_new.3 \
 man/man3/CMS_EncryptedData_decrypt.3 \
 man/man3/CMS_EncryptedData_encrypt.3 \
+man/man3/CMS_EncryptedData_set1_key.3 \
 man/man3/CMS_EnvelopedData_create.3 \
 man/man3/CMS_add0_cert.3 \
 man/man3/CMS_add1_recipient_cert.3 \

doc/man3/CMS_EncryptedData_decrypt.pod

@@ -21,10 +21,10 @@ CMS_EncryptedData_decrypt, CMS_EnvelopedData_decrypt
 =head1 DESCRIPTION
 
 CMS_EncryptedData_decrypt() decrypts a I<cms> EncryptedData object using the
-symmetric I<key> of size I<keylen> bytes. I<out> is a BIO to write the content
-to and I<flags> is an optional set of flags.
-I<dcont> is used in the rare case where the encrypted content is detached. It
-will normally be set to NULL.
+symmetric I<key> of size I<keylen> bytes. AEAD cipher algorithms are not
+supported. I<out> is a BIO to write the content to and I<flags> is an optional
+set of flags. I<dcont> is used in the rare case where the encrypted content is
+detached. It will normally be set to NULL.
 
 The following flags can be passed in the I<flags> parameter.
 

doc/man3/CMS_EncryptedData_encrypt.pod

@@ -34,7 +34,7 @@ B<CMS_PARTIAL>. Internally CMS_final() is called unless B<CMS_STREAM> and/or
 B<CMS_PARTIAL> is specified.
 
 The algorithm passed in the I<cipher> parameter must support ASN1 encoding of
-its parameters.
+its parameters. AEAD cipher algorithms are not supported.
 
 The B<CMS_ContentInfo> structure can be freed using L<CMS_ContentInfo_free(3)>.
 

doc/man3/CMS_EncryptedData_set1_key.pod

@@ -0,0 +1,39 @@
+=pod
+
+=head1 NAME
+
+CMS_EncryptedData_set1_key - Sets the cipher and key for
+CMS EncryptedData
+
+=head1 SYNOPSIS
+
+ #include <openssl/cms.h>
+
+ int CMS_EncryptedData_set1_key(CMS_ContentInfo *cms, const EVP_CIPHER *ciph,
+                                const unsigned char *key, size_t keylen);
+
+=head1 DESCRIPTION
+
+CMS_EncryptedData_set1_key() takes in a I<cms> EncryptedData object and sets
+the appropriate attributes to I<ciph>, it makes a copy of the symmetric I<key>
+of size I<keylen>. AEAD cipher algorithms are not supported.
+
+=head1 RETURN VALUES
+
+CMS_EncryptedData_set1_key() returns 0 if an error occurred otherwise
+returns 1.
+
+=head1 SEE ALSO
+
+L<CMS_EncryptedData_encrypt(3)>, L<CMS_EncryptedData_decrypt(3)>
+
+=head1 COPYRIGHT
+
+Copyright 2025 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut

test/cmsapitest.c

@@ -405,6 +405,78 @@ end:
     return ret;
 }
 
+static int test_encrypted_data(void)
+{
+    const char *msg = "Hello world";
+    BIO *msgbio = BIO_new_mem_buf(msg, (int)strlen(msg));
+    uint8_t key[16] = {0};
+    size_t keylen = 16;
+    CMS_ContentInfo *cms;
+    BIO *decryptbio = BIO_new(BIO_s_mem());
+    char buf[80];
+    int ret = 0;
+
+    cms = CMS_EncryptedData_encrypt(msgbio, EVP_aes_128_cbc(), key, keylen, SMIME_BINARY);
+    if (!TEST_ptr(cms))
+        goto end;
+
+    if (!TEST_true(CMS_EncryptedData_decrypt(cms, key, keylen, NULL, decryptbio, SMIME_BINARY)))
+        goto end;
+
+    /* Check we got the message we first started with */
+    if (!TEST_int_eq(BIO_gets(decryptbio, buf, sizeof(buf)), (int)strlen(msg))
+            || !TEST_int_eq(strcmp(buf, msg), 0))
+        goto end;
+
+    ret = 1;
+end:
+    CMS_ContentInfo_free(cms);
+    BIO_free(msgbio);
+    BIO_free(decryptbio);
+    return ret;
+}
+
+static int test_encrypted_data_aead(void)
+{
+    const char *msg = "Hello world";
+    BIO *msgbio = BIO_new_mem_buf(msg, (int)strlen(msg));
+    uint8_t key[16] = {0};
+    size_t keylen = 16;
+    CMS_ContentInfo *cms;
+    BIO *decryptbio = BIO_new(BIO_s_mem());
+    int ret = 0;
+
+    cms = CMS_ContentInfo_new();
+    if (!TEST_ptr(cms))
+        goto end;
+
+    /*
+     * AEAD algorithms are not supported by the CMS EncryptedData so setting
+     * the cipher to AES GCM 128 will result in a failure
+     */
+    if (!TEST_false(CMS_EncryptedData_set1_key(cms, EVP_aes_128_gcm(), key, keylen)))
+        goto end;
+
+    CMS_ContentInfo_free(cms);
+    cms = NULL;
+
+    /*
+     * AEAD algorithms are not supported by the CMS EncryptedData so setting
+     * the cipher to AES GCM 128 will result in a failure
+     */
+    cms = CMS_EncryptedData_encrypt(msgbio, EVP_aes_128_gcm(), key, keylen, SMIME_BINARY);
+    if (!TEST_ptr_null(cms))
+        goto end;
+
+    ret = 1;
+
+end:
+    CMS_ContentInfo_free(cms);
+    BIO_free(msgbio);
+    BIO_free(decryptbio);
+    return ret;
+}
+
 OPT_TEST_DECLARE_USAGE("certfile privkeyfile derfile\n")
 
 int setup_tests(void)
@@ -452,6 +524,8 @@ int setup_tests(void)
     ADD_TEST(test_CMS_add1_cert);
     ADD_TEST(test_d2i_CMS_bio_NULL);
     ADD_TEST(test_CMS_set1_key_mem_leak);
+    ADD_TEST(test_encrypted_data);
+    ADD_TEST(test_encrypted_data_aead);
     ADD_ALL_TESTS(test_d2i_CMS_decode, 2);
     return 1;
 }

util/missingcrypto.txt

@@ -292,7 +292,6 @@ CMAC_Init(3)
 CMAC_Update(3)
 CMAC_resume(3)
 CMS_ContentInfo_it(3)
-CMS_EncryptedData_set1_key(3)
 CMS_ReceiptRequest_it(3)
 CMS_RecipientEncryptedKey_cert_cmp(3)
 CMS_RecipientEncryptedKey_get0_id(3)