Skip to content

[DRAFT] Bugfixes to SymCrypt-OpenSSL for OpenSSL 3.5 #155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mamckee wants to merge 12 commits into
base: main
Choose a base branch
from
Draft

[DRAFT] Bugfixes to SymCrypt-OpenSSL for OpenSSL 3.5 #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
6d6925a
Check output size of RSA encrypt
mamckee Jan 7, 2026
748a46c
Check pbkdf size before passing to SymCrypt
mamckee Jan 7, 2026
fc11ddf
Fix RSA keydup logic to match OpenSSL behavior
mamckee Jan 7, 2026
e7aa117
Allow zero length secrets and salts for HKDF and TLS1_PRF
mamckee Jan 8, 2026
4c9db10
Validate PBKDF salt and iteration count in set param in non-pkcs5 mode
mamckee Jan 8, 2026
b518591
zalloc AES context
mamckee Jan 8, 2026
ffec1c1
Validate digest is not a XOF for x942 DH
mamckee Jan 9, 2026
f3bddd6
Make provider name order match OpenSSL default
mamckee Jan 12, 2026
f88f7e4
Return -1 for ECDSA error
mamckee Jan 12, 2026
e7ef791
Remove references to UINT type from symcrypt
mamckee Jan 13, 2026
0d874ae
Bump minimum required SymCrypt
mamckee Jan 14, 2026
f530679
Undo change to provider function parameter types
mamckee Jan 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ project(SymCrypt-OpenSSL
HOMEPAGE_URL "https://github.com/microsoft/SymCrypt-OpenSSL")

set(SYMCRYPT_MINIMUM_MAJOR "103")
set(SYMCRYPT_MINIMUM_MINOR "6")
set(SYMCRYPT_MINIMUM_MINOR "8")

find_package(OpenSSL REQUIRED)

Expand All @@ -23,7 +23,7 @@ else()
if (SYMCRYPT_FOUND)
message(STATUS "SymCrypt Includes: ${SYMCRYPT_INCLUDE_DIRS}")
include_directories(${SYMCRYPT_INCLUDE_DIRS})
endif()
endif()
endif()

# Try to find installed SymCrypt
Expand All @@ -34,7 +34,7 @@ else()
if (NOT SYMCRYPT_HEADER)
message(FATAL_ERROR "SymCrypt header file not found. Please set SYMCRYPT_ROOT_DIR or install SymCrypt headers.")
endif()

find_library(SYMCRYPT_LIBRARY symcrypt PATHS ${CMAKE_SOURCE_DIR})
if (SYMCRYPT_LIBRARY)
set(SYMCRYPT_FOUND TRUE)
Expand Down
4 changes: 2 additions & 2 deletions KeysInUse/keysinuse.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ static void keysinuse_init_internal()
}

if ((pthreadErr = pthread_atfork(keysinuse_atfork_prepare,
keysinuse_atfork_parent,
keysinuse_atfork_parent,
keysinuse_atfork_child)) != 0)
{
keysinuse_log_error("Failed to register logging fork handler,SYS_%d", pthreadErr);
Expand Down Expand Up @@ -591,7 +591,7 @@ unsigned int keysinuse_derive_key_identifier(_In_reads_bytes_(cbEncodedKey) cons
_Out_writes_bytes_opt_(cbEncodedKey)char *pbKeyIdentifier, unsigned long cbKeyIdentifier)
{
BYTE abHash[SYMCRYPT_SHA256_RESULT_SIZE];
UINT cbHash = SYMCRYPT_SHA256_RESULT_SIZE;
unsigned int cbHash = SYMCRYPT_SHA256_RESULT_SIZE;

if (pbKeyIdentifier == NULL)
{
Expand Down
4 changes: 2 additions & 2 deletions ScosslCommon/inc/scossl_rsa.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@ SCOSSL_STATUS scossl_rsapss_verify(_In_ PSYMCRYPT_RSAKEY key, int mdnid, int cbS
_In_reads_bytes_(cbHashValue) PCBYTE pbHashValue, SIZE_T cbHashValue,
_In_reads_bytes_(pcbSignature) PCBYTE pbSignature, SIZE_T pcbSignature);

SCOSSL_STATUS scossl_rsa_encrypt(_In_ PSYMCRYPT_RSAKEY key, UINT padding,
SCOSSL_STATUS scossl_rsa_encrypt(_In_ PSYMCRYPT_RSAKEY key, UINT8 padding,
int mdnid, _In_reads_bytes_opt_(cbLabel) PCBYTE pbLabel, SIZE_T cbLabel,
_In_reads_bytes_(cbSrc) PCBYTE pbSrc, SIZE_T cbSrc,
_Out_writes_bytes_(*pcbDst) PBYTE pbDst, _Out_ INT32 *pcbDst, SIZE_T cbDst);

SCOSSL_STATUS scossl_rsa_decrypt(_In_ PSYMCRYPT_RSAKEY key, UINT padding,
SCOSSL_STATUS scossl_rsa_decrypt(_In_ PSYMCRYPT_RSAKEY key, UINT8 padding,
int mdnid, _In_reads_bytes_opt_(cbLabel) PCBYTE pbLabel, SIZE_T cbLabel,
_In_reads_bytes_(cbSrc) PCBYTE pbSrc, SIZE_T cbSrc,
_Out_writes_bytes_(*pcbDst) PBYTE pbDst, _Out_ INT32 *pcbDst, SIZE_T cbDst);
Expand Down
4 changes: 2 additions & 2 deletions ScosslCommon/src/scossl_rsa.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ SCOSSL_STATUS scossl_rsapss_verify(PSYMCRYPT_RSAKEY key, int mdnid, int cbSalt,
}

_Use_decl_annotations_
SCOSSL_STATUS scossl_rsa_encrypt(PSYMCRYPT_RSAKEY key, UINT padding,
SCOSSL_STATUS scossl_rsa_encrypt(PSYMCRYPT_RSAKEY key, UINT8 padding,
int mdnid, PCBYTE pbLabel, SIZE_T cbLabel, // OAEP-only parameters
PCBYTE pbSrc, SIZE_T cbSrc,
PBYTE pbDst, INT32 *pcbDst, SIZE_T cbDst)
Expand Down Expand Up @@ -545,7 +545,7 @@ SCOSSL_STATUS scossl_rsa_encrypt(PSYMCRYPT_RSAKEY key, UINT padding,
}

_Use_decl_annotations_
SCOSSL_STATUS scossl_rsa_decrypt(PSYMCRYPT_RSAKEY key, UINT padding,
SCOSSL_STATUS scossl_rsa_decrypt(PSYMCRYPT_RSAKEY key, UINT8 padding,
int mdnid, PCBYTE pbLabel, SIZE_T cbLabel, // OAEP-only parameters
PCBYTE pbSrc, SIZE_T cbSrc,
PBYTE pbDst, INT32 *pcbDst, SIZE_T cbDst)
Expand Down
11 changes: 9 additions & 2 deletions SymCryptProvider/src/asymcipher/p_scossl_rsa_cipher.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ typedef struct
OSSL_LIB_CTX *libctx;

SCOSSL_PROV_RSA_KEY_CTX *keyCtx;
UINT padding;
UINT8 padding;
int operation;

// OAEP Parameters
Expand Down Expand Up @@ -145,6 +145,13 @@ static SCOSSL_STATUS p_scossl_rsa_cipher_encrypt(_In_ SCOSSL_RSA_CIPHER_CTX *ctx
return SCOSSL_FAILURE;
}

if (out != NULL &&
outsize < SymCryptRsakeySizeofModulus(ctx->keyCtx->key))
{
ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
return SCOSSL_FAILURE;
}

if (ctx->operation != EVP_PKEY_OP_ENCRYPT)
{
ERR_raise(ERR_LIB_PROV, ERR_R_OPERATION_FAIL);
Expand Down Expand Up @@ -305,7 +312,7 @@ static SCOSSL_STATUS p_scossl_rsa_cipher_set_ctx_params(_Inout_ SCOSSL_RSA_CIPHE
// Padding mode may be passed as legacy NID or string, and is
// checked against the padding modes the ScOSSL provider supports
int i = 0;
UINT padding;
unsigned int padding;

switch (p->data_type)
{
Expand Down
13 changes: 5 additions & 8 deletions SymCryptProvider/src/ciphers/p_scossl_aes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ typedef struct
SIZE_T cbBuf;

OSSL_LIB_CTX *libctx;
UINT tlsVersion;
UINT32 tlsVersion;
PBYTE tlsMac;
SIZE_T tlsMacSize;

Expand Down Expand Up @@ -154,7 +154,7 @@ static SCOSSL_STATUS p_scossl_aes_generic_decrypt_init(_Inout_ SCOSSL_AES_CTX *c
// the unpadded record, and saves the result to ctx->tlsMac.
//
// The MAC will later be fetched through p_scossl_aes_generic_get_ctx_params
// This function is adapted from ssl3_cbc_copy_mac in ssl/record/tls_pad.c, and
// This function is adapted from ssl3_cbc_copy_mac in ssl/record/tls_pad.c, and
// SymCryptTlsCbcHmacVerifyCore from SymCrypt, and runs in constant time w.r.t
// the values in pbData. In case of bad padding, a random MAC is assigned instead
static SCOSSL_STATUS p_scossl_aes_tls_remove_padding_and_copy_mac(
Expand Down Expand Up @@ -230,7 +230,7 @@ static SCOSSL_STATUS p_scossl_aes_tls_remove_padding_and_copy_mac(
macStart = macEnd - ctx->tlsMacSize;

rotatedMac = rotatedMacBuf + ((0 - (SIZE_T)rotatedMacBuf) & 0x3f);

// Find and extract MAC, and verify padding
memset(rotatedMac, 0, ctx->tlsMacSize);
for (i = 0, j = 0; i < cbTail-1; i++)
Expand Down Expand Up @@ -745,7 +745,7 @@ static SCOSSL_STATUS p_scossl_aes_generic_set_ctx_params(_Inout_ SCOSSL_AES_CTX

if ((p = OSSL_PARAM_locate_const(params, OSSL_CIPHER_PARAM_TLS_VERSION)) != NULL)
{
UINT tlsVersion;
unsigned int tlsVersion;
if (!OSSL_PARAM_get_uint(p, &tlsVersion))
{
ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER);
Expand Down Expand Up @@ -994,16 +994,13 @@ static SCOSSL_STATUS scossl_aes_cfb8_cipher(_Inout_ SCOSSL_AES_CTX *ctx,
#define IMPLEMENT_SCOSSL_AES_GENERIC_CIPHER(kbits, ivlen, lcmode, UCMODE, type, blocksize) \
SCOSSL_AES_CTX *p_scossl_aes_##kbits##_##lcmode##_newctx(_In_ SCOSSL_PROVCTX *provctx) \
{ \
SCOSSL_COMMON_ALIGNED_ALLOC(ctx, OPENSSL_malloc, SCOSSL_AES_CTX); \
SCOSSL_COMMON_ALIGNED_ALLOC(ctx, OPENSSL_zalloc, SCOSSL_AES_CTX); \
if (ctx != NULL) \
{ \
ctx->keylen = kbits >> 3; \
ctx->pad = TRUE; \
ctx->cipher = (OSSL_FUNC_cipher_cipher_fn *)&scossl_aes_##lcmode##_cipher; \
ctx->libctx = provctx->libctx; \
ctx->tlsMac = NULL; \
ctx->tlsMacSize = 0; \
ctx->tlsVersion = 0; \
} \
\
return ctx; \
Expand Down
70 changes: 30 additions & 40 deletions SymCryptProvider/src/kdf/p_scossl_hkdf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,36 +278,26 @@ SCOSSL_STATUS p_scossl_hkdf_set_ctx_params(_Inout_ SCOSSL_PROV_HKDF_CTX *ctx, co

if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_SALT)) != NULL)
{
PBYTE pbSalt = NULL;
SIZE_T cbSalt = 0;
OPENSSL_clear_free(ctx->hkdfCtx->pbSalt, ctx->hkdfCtx->cbSalt);
ctx->hkdfCtx->pbSalt = NULL;

if (p->data_size > 0 &&
!OSSL_PARAM_get_octet_string(p, (void **)&pbSalt, 0, &cbSalt))
if (!OSSL_PARAM_get_octet_string(p, (void **)&ctx->hkdfCtx->pbSalt, 0, &ctx->hkdfCtx->cbSalt))
{
ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER);
return SCOSSL_FAILURE;
}

OPENSSL_clear_free(ctx->hkdfCtx->pbSalt, ctx->hkdfCtx->cbSalt);
ctx->hkdfCtx->pbSalt = pbSalt;
ctx->hkdfCtx->cbSalt = cbSalt;
}

if ((p = OSSL_PARAM_locate_const(params, OSSL_KDF_PARAM_KEY)) != NULL)
{
PBYTE pbKey = NULL;
SIZE_T cbKey = 0;
OPENSSL_clear_free(ctx->hkdfCtx->pbKey, ctx->hkdfCtx->cbKey);
ctx->hkdfCtx->pbKey = NULL;

if (p->data_size > 0 &&
!OSSL_PARAM_get_octet_string(p, (void **)&pbKey, 0, &cbKey))
if (!OSSL_PARAM_get_octet_string(p, (void **)&ctx->hkdfCtx->pbKey, 0, &ctx->hkdfCtx->cbKey))
{
ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_GET_PARAMETER);
return SCOSSL_FAILURE;
}

OPENSSL_clear_free(ctx->hkdfCtx->pbKey, ctx->hkdfCtx->cbKey);
ctx->hkdfCtx->pbKey = pbKey;
ctx->hkdfCtx->cbKey = cbKey;
}

// Parameters may contain multiple info params that must all be processed
Expand Down Expand Up @@ -420,49 +410,49 @@ SCOSSL_STATUS p_scossl_hkdf_derive(_In_ SCOSSL_PROV_HKDF_CTX *ctx,
*
*/
static
SCOSSL_STATUS p_scossl_tls13_hkdf_expand(_In_ SCOSSL_HKDF_CTX *ctx,
SCOSSL_STATUS p_scossl_tls13_hkdf_expand(_In_ SCOSSL_HKDF_CTX *ctx,
_Out_writes_bytes_(keylen) unsigned char *key, size_t keylen)
{
SYMCRYPT_ERROR scError = SYMCRYPT_NO_ERROR;
PCSYMCRYPT_MAC symcryptHmacAlg = NULL;
SYMCRYPT_HKDF_EXPANDED_KEY scExpandedKey;
SIZE_T labelLen = 0;
SIZE_T totalLen = 0;

BYTE hkdflabel[HKDF_MAXBUF];
SIZE_T hkdflabellen = 0;
if (ctx->md == NULL)

if (ctx->md == NULL)
{
ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_MESSAGE_DIGEST);
return SCOSSL_FAILURE;
}

if (ctx->pbKey == NULL)
if (ctx->pbKey == NULL)
{
ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_KEY);
return SCOSSL_FAILURE;
}

symcryptHmacAlg = scossl_get_symcrypt_hmac_algorithm(EVP_MD_type(ctx->md));
if (symcryptHmacAlg == NULL)
if (symcryptHmacAlg == NULL)
{
return SCOSSL_FAILURE;
}

labelLen = ctx->cbPrefix + ctx->cbLabel;

// Ensure this value does not exceed 0xFF, as only the least-significant byte is copied into hkdflabel.
// If the value exceeds 0xFF, it will overflow and corrupt the label encoding.
if (labelLen > 0xFF)
{
ERR_raise(ERR_LIB_PROV, PROV_R_LENGTH_TOO_LARGE);
return SCOSSL_FAILURE;
}

// 2 bytes for output length, 1 byte for label length, and 1 byte for context length
totalLen = 2 + 1 + labelLen + 1 + ctx->cbData;
if (totalLen > HKDF_MAXBUF)
if (totalLen > HKDF_MAXBUF)
{
ERR_raise(ERR_LIB_PROV, PROV_R_LENGTH_TOO_LARGE);
return SCOSSL_FAILURE;
Expand All @@ -480,7 +470,7 @@ SCOSSL_STATUS p_scossl_tls13_hkdf_expand(_In_ SCOSSL_HKDF_CTX *ctx,
hkdflabellen += ctx->cbPrefix;
memcpy(hkdflabel + hkdflabellen, ctx->pbLabel, ctx->cbLabel);
hkdflabellen += ctx->cbLabel;

if (ctx->cbData > 0xFF)
{
ERR_raise(ERR_LIB_PROV, PROV_R_LENGTH_TOO_LARGE);
Expand All @@ -490,7 +480,7 @@ SCOSSL_STATUS p_scossl_tls13_hkdf_expand(_In_ SCOSSL_HKDF_CTX *ctx,
hkdflabel[hkdflabellen++] = (BYTE)ctx->cbData;

// Context
if (ctx->cbData > 0)
if (ctx->cbData > 0)
{
memcpy(hkdflabel + hkdflabellen, ctx->pbData, ctx->cbData);
hkdflabellen += ctx->cbData;
Expand Down Expand Up @@ -519,32 +509,32 @@ SCOSSL_STATUS p_scossl_tls13_hkdf_expand(_In_ SCOSSL_HKDF_CTX *ctx,
}

static
SCOSSL_STATUS p_scossl_tls13kdf_generate_secret(_In_ SCOSSL_HKDF_CTX *ctx,
SCOSSL_STATUS p_scossl_tls13kdf_generate_secret(_In_ SCOSSL_HKDF_CTX *ctx,
_Out_writes_bytes_(keylen) unsigned char *key, size_t keylen)
{
SYMCRYPT_ERROR scError = SYMCRYPT_NO_ERROR;
SCOSSL_STATUS status = SCOSSL_FAILURE;
PCSYMCRYPT_MAC symcryptHmacAlg = NULL;
BYTE *default_zeros = NULL;
BYTE empty_hash[EVP_MAX_MD_SIZE];
BYTE empty_hash[EVP_MAX_MD_SIZE];
BYTE expanded_secret[EVP_MAX_MD_SIZE];
SCOSSL_HKDF_CTX *dupCtx;
SIZE_T mdlen = 0;
PBYTE pbSavedKey;
SIZE_T cbSavedKey = 0;

if (ctx == NULL || ctx->md == NULL)
{
ERR_raise(ERR_LIB_PROV, PROV_R_MISSING_MESSAGE_DIGEST);
return SCOSSL_FAILURE;
}

symcryptHmacAlg = scossl_get_symcrypt_hmac_algorithm(EVP_MD_type(ctx->md));
if (symcryptHmacAlg == NULL)
if (symcryptHmacAlg == NULL)
{
return SCOSSL_FAILURE;
}

mdlen = EVP_MD_get_size(ctx->md);
if (mdlen <= 0)
{
Expand All @@ -558,21 +548,21 @@ SCOSSL_STATUS p_scossl_tls13kdf_generate_secret(_In_ SCOSSL_HKDF_CTX *ctx,
ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
return SCOSSL_FAILURE;
}

default_zeros = OPENSSL_zalloc(EVP_MAX_MD_SIZE);
if (default_zeros == NULL)
{
ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
goto cleanup;
}
if (dupCtx->pbKey == NULL)

if (dupCtx->pbKey == NULL)
{
dupCtx->pbKey = default_zeros;
dupCtx->cbKey = mdlen;
}

if (dupCtx->pbSalt == NULL)
if (dupCtx->pbSalt == NULL)
{
dupCtx->pbSalt = default_zeros;
dupCtx->cbSalt = mdlen;
Expand All @@ -596,7 +586,7 @@ SCOSSL_STATUS p_scossl_tls13kdf_generate_secret(_In_ SCOSSL_HKDF_CTX *ctx,
dupCtx->pbKey = dupCtx->pbSalt;
dupCtx->cbKey = dupCtx->cbSalt;

if (SCOSSL_SUCCESS != p_scossl_tls13_hkdf_expand(dupCtx, expanded_secret, keylen))
if (SCOSSL_SUCCESS != p_scossl_tls13_hkdf_expand(dupCtx, expanded_secret, keylen))
{
goto cleanup;
}
Expand All @@ -613,7 +603,7 @@ SCOSSL_STATUS p_scossl_tls13kdf_generate_secret(_In_ SCOSSL_HKDF_CTX *ctx,
dupCtx->pbKey, dupCtx->cbKey,
dupCtx->pbSalt, dupCtx->cbSalt,
key, keylen);
if (scError != SYMCRYPT_NO_ERROR)
if (scError != SYMCRYPT_NO_ERROR)
{
SCOSSL_PROV_LOG_SYMCRYPT_ERROR("SymCryptHkdfExtractPrk failed", scError);
goto cleanup;
Expand All @@ -636,7 +626,7 @@ SCOSSL_STATUS p_scossl_tls13kdf_derive(_In_ SCOSSL_PROV_HKDF_CTX *ctx,
ERR_raise(ERR_LIB_PROV, ERR_R_PASSED_NULL_PARAMETER);
return SCOSSL_FAILURE;
}

if (!p_scossl_tls13kdf_set_ctx_params(ctx, params))
{
ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER);
Expand Down
2 changes: 1 addition & 1 deletion SymCryptProvider/src/kdf/p_scossl_kbkdf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ typedef struct
SIZE_T cbLabel;
PCSYMCRYPT_MAC pMac;

UINT macType;
UINT8 macType;
SIZE_T cbCmacKey;
const SCOSSL_KMAC_EXTENSIONS *pMacEx;
} SCOSSL_PROV_KBKDF_CTX;
Expand Down
Loading