return ssl alert unrecognized_name when https domain not registered (#3620)

pkg/util/tcpmux/httpconnect.go

@@ -40,7 +40,8 @@ func NewHTTPConnectTCPMuxer(listener net.Listener, passthrough bool, timeout tim
 	ret := &HTTPConnectTCPMuxer{passthrough: passthrough}
 	mux, err := vhost.NewMuxer(listener, ret.getHostFromHTTPConnect, timeout)
 	mux.SetCheckAuthFunc(ret.auth).
-		SetSuccessHookFunc(ret.sendConnectResponse)
+		SetSuccessHookFunc(ret.sendConnectResponse).
+		SetFailHookFunc(vhostFailed)
 	ret.Muxer = mux
 	return ret, err
 }
@@ -92,6 +93,15 @@ func (muxer *HTTPConnectTCPMuxer) auth(c net.Conn, username, password string, re
 	return false, nil
 }
 
+func vhostFailed(c net.Conn) {
+	res := vhost.NotFoundResponse()
+	if res.Body != nil {
+		defer res.Body.Close()
+	}
+	_ = res.Write(c)
+	_ = c.Close()
+}
+
 func (muxer *HTTPConnectTCPMuxer) getHostFromHTTPConnect(c net.Conn) (net.Conn, map[string]string, error) {
 	reqInfoMap := make(map[string]string, 0)
 	sc, rd := libnet.NewSharedConn(c)