SERVER-36942 Differentiate invalid hostname from invalid certificate

Author: sgolemon-corp

src/mongo/util/net/ssl/apple.hpp

@@ -89,6 +89,7 @@ struct Context {
     ::SSLProtocol protoMin = kTLSProtocol1;
     ::SSLProtocol protoMax = kTLSProtocol12;
     CFUniquePtr<::CFArrayRef> certs;
+    bool allowInvalidHostnames = false;
 };
 
 }  // namespace apple

src/mongo/util/net/ssl/detail/impl/engine_apple.ipp

@@ -124,6 +124,9 @@ engine::engine(context::native_handle_type context, const std::string& remoteHos
         }
         _protoMin = context->protoMin;
         _protoMax = context->protoMax;
+        if (context->allowInvalidHostnames) {
+            _remoteHostName.clear();
+        }
     } else {
         apple::Context def;
         _protoMin = def.protoMin;

src/mongo/util/net/ssl_manager_apple.cpp

@@ -1197,6 +1197,9 @@ StatusWith<std::pair<::SSLProtocol, ::SSLProtocol>> parseProtocolRange(const SSL
 Status SSLManagerApple::initSSLContext(asio::ssl::apple::Context* context,
                                        const SSLParams& params,
                                        ConnectionDirection direction) {
+    // Options.
+    context->allowInvalidHostnames = _allowInvalidHostnames;
+
     // Protocol Version.
     const auto swProto = parseProtocolRange(params);
     if (!swProto.isOK()) {
@@ -1367,9 +1370,7 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManagerApple::parseAndValidatePeerCe
     auto result = ::kSecTrustResultInvalid;
     uassertOSStatusOK(::SecTrustEvaluate(cftrust.get(), &result), ErrorCodes::SSLHandshakeFailed);
     if ((result != ::kSecTrustResultProceed) && (result != ::kSecTrustResultUnspecified)) {
-        const bool proceed = _allowInvalidCertificates ||
-            (_allowInvalidHostnames && (result == ::kSecTrustResultRecoverableTrustFailure));
-        return badCert(explainTrustFailure(cftrust.get(), result), proceed);
+        return badCert(explainTrustFailure(cftrust.get(), result), _allowInvalidCertificates);
     }
 
     auto cert = ::SecTrustGetCertificateAtIndex(cftrust.get(), 0);