Update pre10

include/internal/tsan_assist.h

@@ -8,8 +8,9 @@
  */
  
 /*
- * Goal here is to facilitate writing "thread-opportunistic" code that
- * withstands Thread Sanitizer's scrutiny. "Thread-opportunistic" is when
+ * Contemporary compilers implement lock-free atomic memory access
+ * primitives that facilitate writing "thread-opportunistic" or even real
+ * multi-threading low-overhead code. "Thread-opportunistic" is when
  * exact result is not required, e.g. some statistics, or execution flow
  * doesn't have to be unambiguous. Simplest example is lazy "constant"
  * initialization when one can synchronize on variable itself, e.g.
@@ -28,12 +29,22 @@
  * bother. Having Thread Sanitizer accept "thread-opportunistic" code
  * allows to move on trouble-shooting real bugs.
  *
- * We utilize the fact that compilers that implement Thread Sanitizer
- * implement even atomic operations. Then it's assumed that
- * ATOMIC_{LONG|INT}_LOCK_FREE are assigned same value as
- * ATOMIC_POINTER_LOCK_FREE. And check for >= 2 ensures that correspodning
+ * Resolving Thread Sanitizer nits was the initial purpose for this module,
+ * but it was later extended with more nuanced primitives that are useful
+ * even in "non-opportunistic" scenarios. Most notably verifying if a shared
+ * structure is fully initialized and bypassing the initialization lock.
+ * It's suggested to view macros defined in this module as "annotations" for
+ * thread-safe lock-free code, "Thread-Safe ANnotations"...
+ *
+ * It's assumed that ATOMIC_{LONG|INT}_LOCK_FREE are assigned same value as
+ * ATOMIC_POINTER_LOCK_FREE. And check for >= 2 ensures that corresponding
  * code is inlined. It should be noted that statistics counters become
  * accurate in such case.
+ *
+ * Special note about TSAN_QUALIFIER. It might be undesired to use it in
+ * a shared header. Because whether operation on specific variable or member
+ * is atomic or not might be irrelevant in other modules. In such case one
+ * can use TSAN_QUALIFIER in cast specifically when it has to count.
  */
 
 #if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \
@@ -46,6 +57,8 @@
 #  define tsan_load(ptr) atomic_load_explicit((ptr), memory_order_relaxed)
 #  define tsan_store(ptr, val) atomic_store_explicit((ptr), (val), memory_order_relaxed)
 #  define tsan_counter(ptr) atomic_fetch_add_explicit((ptr), 1, memory_order_relaxed)
+#  define tsan_ld_acq(ptr) atomic_load_explicit((ptr), memory_order_acquire)
+#  define tsan_st_rel(ptr, val) atomic_store_explicit((ptr), (val), memory_order_release)
 # endif
 
 #elif defined(__GNUC__) && defined(__ATOMIC_RELAXED)
@@ -56,21 +69,57 @@
 #  define tsan_load(ptr) __atomic_load_n((ptr), __ATOMIC_RELAXED)
 #  define tsan_store(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELAXED)
 #  define tsan_counter(ptr) __atomic_fetch_add((ptr), 1, __ATOMIC_RELAXED)
+#  define tsan_ld_acq(ptr) __atomic_load_n((ptr), __ATOMIC_ACQUIRE)
+#  define tsan_st_rel(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELEASE)
 # endif
 
-#elif defined(_MSC_VER) && _MSC_VER>=1200
-
+#elif defined(_MSC_VER) && _MSC_VER>=1200 \
+      && (defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \
+          defined(_M_ARM64) || (defined(_M_ARM) && _M_ARM >= 7))
+/*
+ * There is subtle dependency on /volatile:<iso|ms> command-line option.
+ * "ms" implies same semantic as memory_order_acquire for loads and
+ * memory_order_release for stores, while "iso" - memory_order_relaxed for
+ * either. Real complication is that defaults are different on x86 and ARM.
+ * There is explanation for that, "ms" is backward compatible with earlier
+ * compiler versions, while multi-processor ARM can be viewed as brand new
+ * platform to MSC and its users, and with non-relaxed semantic taking toll
+ * with additional instructions and penalties, it kind of makes sense to
+ * default to "iso"...
+ */
 # define TSAN_QUALIFIER volatile
-# define tsan_load(ptr) (*(ptr))
-# define tsan_store(ptr, val) (*(ptr) = (val))
+# if defined(_M_ARM) || defined(_M_ARM64)
+#  define _InterlockedExchangeAdd _InterlockedExchangeAdd_nf
+#  pragma intrinsic(_InterlockedExchangeAdd_nf)
+#  pragma intrinsic(__iso_volatile_load32, __iso_volatile_store32)
+#  ifdef _WIN64
+#   define _InterlockedExchangeAdd64 _InterlockedExchangeAdd64_nf
+#   pragma intrinsic(_InterlockedExchangeAdd64_nf)
+#   pragma intrinsic(__iso_volatile_load64, __iso_volatile_store64)
+#   define tsan_load(ptr) (sizeof(*(ptr)) == 8 ? __iso_volatile_load64(ptr) \
+                                               : __iso_volatile_load32(ptr))
+#   define tsan_store(ptr, val) (sizeof(*(ptr)) == 8 ? __iso_volatile_store64((ptr), (val)) \
+                                                     : __iso_volatile_store32((ptr), (val)))
+#  else
+#   define tsan_load(ptr) __iso_volatile_load32(ptr)
+#   define tsan_store(ptr, val) __iso_volatile_store32((ptr), (val))
+#  endif
+# else
+#  define tsan_load(ptr) (*(ptr))
+#  define tsan_store(ptr, val) (*(ptr) = (val))
+# endif
 # pragma intrinsic(_InterlockedExchangeAdd)
 # ifdef _WIN64
 #  pragma intrinsic(_InterlockedExchangeAdd64)
-#  define tsan_counter(ptr) (sizeof(*ptr) == 8 ? _InterlockedExchangeAdd64((ptr), 1) \
-                                               : _InterlockedExchangeAdd((ptr), 1))
+#  define tsan_counter(ptr) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), 1) \
+                                                 : _InterlockedExchangeAdd((ptr), 1))
 # else
 #  define tsan_counter(ptr) _InterlockedExchangeAdd((ptr), 1)
 # endif
+# if !defined(_ISO_VOLATILE)
+#  define tsan_ld_acq(ptr) (*(ptr))
+#  define tsan_st_rel(ptr, val) (*(ptr) = (val))
+# endif
 
 #endif
 
@@ -80,5 +129,10 @@
 # define tsan_load(ptr) (*(ptr))
 # define tsan_store(ptr, val) (*(ptr) = (val))
 # define tsan_counter(ptr) ((*(ptr))++)
+/*
+ * Lack of tsan_ld_acq and tsan_ld_rel means that compiler support is not
+ * sophisticated enough to support them. Code that relies on them should be
+ * protected with #ifdef tsan_ld_acq with locked fallback.
+ */
 
 #endif

include/openssl/ec.h

@@ -1429,6 +1429,19 @@ void EC_KEY_METHOD_get_verify(const EC_KEY_METHOD *meth,
                                 EVP_PKEY_OP_DERIVE, \
                                 EVP_PKEY_CTRL_GET_EC_KDF_UKM, 0, (void *)(p))
 
+/* SM2 will skip the operation check so no need to pass operation here */
+# define EVP_PKEY_CTX_set1_id(ctx, id, id_len) \
+        EVP_PKEY_CTX_ctrl(ctx, -1, -1, \
+                                EVP_PKEY_CTRL_SET1_ID, (int)id_len, (void*)(id))
+
+# define EVP_PKEY_CTX_get1_id(ctx, id) \
+        EVP_PKEY_CTX_ctrl(ctx, -1, -1, \
+                                EVP_PKEY_CTRL_GET1_ID, 0, (void*)(id))
+
+# define EVP_PKEY_CTX_get1_id_len(ctx, id_len) \
+        EVP_PKEY_CTX_ctrl(ctx, -1, -1, \
+                                EVP_PKEY_CTRL_GET1_ID_LEN, 0, (void*)(id_len))
+
 # define EVP_PKEY_CTRL_EC_PARAMGEN_CURVE_NID             (EVP_PKEY_ALG_CTRL + 1)
 # define EVP_PKEY_CTRL_EC_PARAM_ENC                      (EVP_PKEY_ALG_CTRL + 2)
 # define EVP_PKEY_CTRL_EC_ECDH_COFACTOR                  (EVP_PKEY_ALG_CTRL + 3)
@@ -1439,6 +1452,9 @@ void EC_KEY_METHOD_get_verify(const EC_KEY_METHOD *meth,
 # define EVP_PKEY_CTRL_GET_EC_KDF_OUTLEN                 (EVP_PKEY_ALG_CTRL + 8)
 # define EVP_PKEY_CTRL_EC_KDF_UKM                        (EVP_PKEY_ALG_CTRL + 9)
 # define EVP_PKEY_CTRL_GET_EC_KDF_UKM                    (EVP_PKEY_ALG_CTRL + 10)
+# define EVP_PKEY_CTRL_SET1_ID                           (EVP_PKEY_ALG_CTRL + 11)
+# define EVP_PKEY_CTRL_GET1_ID                           (EVP_PKEY_ALG_CTRL + 12)
+# define EVP_PKEY_CTRL_GET1_ID_LEN                       (EVP_PKEY_ALG_CTRL + 13)
 /* KDF types */
 # define EVP_PKEY_ECDH_KDF_NONE                          1
 # define EVP_PKEY_ECDH_KDF_X9_62                         2

include/openssl/evp.h

@@ -180,6 +180,7 @@ int (*EVP_MD_meth_get_ctrl(const EVP_MD *md))(EVP_MD_CTX *ctx, int cmd,
  * if the following flag is set.
  */
 # define EVP_MD_CTX_FLAG_FINALISE        0x0200
+/* NOTE: 0x0400 is reserved for internal usage in evp_int.h */
 
 EVP_CIPHER *EVP_CIPHER_meth_new(int cipher_type, int block_size, int key_len);
 EVP_CIPHER *EVP_CIPHER_meth_dup(const EVP_CIPHER *cipher);
@@ -453,6 +454,7 @@ void EVP_MD_CTX_set_update_fn(EVP_MD_CTX *ctx,
 # define EVP_MD_CTX_block_size(e)        EVP_MD_block_size(EVP_MD_CTX_md(e))
 # define EVP_MD_CTX_type(e)              EVP_MD_type(EVP_MD_CTX_md(e))
 EVP_PKEY_CTX *EVP_MD_CTX_pkey_ctx(const EVP_MD_CTX *ctx);
+void EVP_MD_CTX_set_pkey_ctx(EVP_MD_CTX *ctx, EVP_PKEY_CTX *pctx);
 void *EVP_MD_CTX_md_data(const EVP_MD_CTX *ctx);
 
 int EVP_CIPHER_nid(const EVP_CIPHER *cipher);
@@ -1519,6 +1521,10 @@ void EVP_PKEY_meth_set_public_check(EVP_PKEY_METHOD *pmeth,
 void EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
                                    int (*check) (EVP_PKEY *pkey));
 
+void EVP_PKEY_meth_set_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (*digest_custom) (EVP_PKEY_CTX *ctx,
+                                                           EVP_MD_CTX *mctx));
+
 void EVP_PKEY_meth_get_init(const EVP_PKEY_METHOD *pmeth,
                             int (**pinit) (EVP_PKEY_CTX *ctx));
 
@@ -1620,6 +1626,9 @@ void EVP_PKEY_meth_get_public_check(const EVP_PKEY_METHOD *pmeth,
 void EVP_PKEY_meth_get_param_check(const EVP_PKEY_METHOD *pmeth,
                                    int (**pcheck) (EVP_PKEY *pkey));
 
+void EVP_PKEY_meth_get_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (**pdigest_custom) (EVP_PKEY_CTX *ctx,
+                                                             EVP_MD_CTX *mctx));
 void EVP_add_alg_module(void);
 
 

include/openssl/hmac.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -14,7 +14,9 @@
 
 # include <openssl/evp.h>
 
-# define HMAC_MAX_MD_CBLOCK      128/* largest known is SHA512 */
+# if OPENSSL_API_COMPAT < 0x10200000L
+#  define HMAC_MAX_MD_CBLOCK      128    /* Deprecated */
+# endif
 
 #ifdef  __cplusplus
 extern "C" {

include/openssl/ocsp.h

@@ -197,6 +197,8 @@ int OCSP_response_status(OCSP_RESPONSE *resp);
 OCSP_BASICRESP *OCSP_response_get1_basic(OCSP_RESPONSE *resp);
 
 const ASN1_OCTET_STRING *OCSP_resp_get0_signature(const OCSP_BASICRESP *bs);
+const X509_ALGOR *OCSP_resp_get0_tbs_sigalg(const OCSP_BASICRESP *bs);
+const OCSP_RESPDATA *OCSP_resp_get0_respdata(const OCSP_BASICRESP *bs);
 int OCSP_resp_get0_signer(OCSP_BASICRESP *bs, X509 **signer,
                           STACK_OF(X509) *extra_certs);
 

include/openssl/ssl.h

@@ -1913,8 +1913,8 @@ void SSL_set0_CA_list(SSL *s, STACK_OF(X509_NAME) *name_list);
 void SSL_CTX_set0_CA_list(SSL_CTX *ctx, STACK_OF(X509_NAME) *name_list);
 __owur const STACK_OF(X509_NAME) *SSL_get0_CA_list(const SSL *s);
 __owur const STACK_OF(X509_NAME) *SSL_CTX_get0_CA_list(const SSL_CTX *ctx);
-__owur int SSL_add1_CA_list(SSL *ssl, const X509 *x);
-__owur int SSL_CTX_add1_CA_list(SSL_CTX *ctx, const X509 *x);
+__owur int SSL_add1_to_CA_list(SSL *ssl, const X509 *x);
+__owur int SSL_CTX_add1_to_CA_list(SSL_CTX *ctx, const X509 *x);
 __owur const STACK_OF(X509_NAME) *SSL_get0_peer_CA_list(const SSL *s);
 
 void SSL_set_client_CA_list(SSL *s, STACK_OF(X509_NAME) *name_list);

include/openssl/x509.h

@@ -660,7 +660,7 @@ int X509_get_signature_type(const X509 *x);
 
 /*
  * This one is only used so that a binary form can output, as in
- * i2d_X509_NAME(X509_get_X509_PUBKEY(x), &buf)
+ * i2d_X509_PUBKEY(X509_get_X509_PUBKEY(x), &buf)
  */
 X509_PUBKEY *X509_get_X509_PUBKEY(const X509 *x);
 const STACK_OF(X509_EXTENSION) *X509_get0_extensions(const X509 *x);