Merge pull request #3454 from fatedier/dev

release v0.49.0

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   go-version-latest:
     docker:
-      - image: cimg/go:1.19-node
+      - image: cimg/go:1.20-node
     resource_class: large
     steps:
       - checkout
@@ -10,7 +10,7 @@ jobs:
       - run: make alltest
   go-version-last:
     docker:
-      - image: cimg/go:1.18-node
+      - image: cimg/go:1.19-node
     resource_class: large
     steps:
       - checkout

.github/pull_request_template.md

@@ -0,0 +1,10 @@
+### Summary
+
+copilot:summary
+
+### WHY
+<!-- author to complete -->
+
+### Walkthrough
+
+copilot:walkthrough

.github/workflows/golangci-lint.yml

@@ -16,13 +16,13 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: '1.20'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.49.0
+          version: v1.51
 
           # Optional: golangci-lint command line arguments.
           # args: --issues-exit-code=0

.github/workflows/goreleaser.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: '1.20'
           
       - name: Make All
         run: |

.golangci.yml

@@ -1,6 +1,5 @@
 service:
-  # When updating this, also update the version stored in docker/build-tools/Dockerfile in the istio/tools repo.
-  golangci-lint-version: 1.49.x # use the fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: 1.51.x # use the fixed version to not introduce new linters unexpectedly
   
 run:
   concurrency: 4
@@ -127,6 +126,11 @@ issues:
         - errcheck
         - maligned
 
+    # keep it until we only support go1.20
+    - linters:
+        - staticcheck
+      text: "SA1019: rand.Seed has been deprecated"
+
   # Independently from option `exclude` we use default exclude patterns,
   # it can be disabled by this option. To list all
   # excluded by default patterns execute `golangci-lint run --help`.

Makefile

@@ -16,6 +16,12 @@ file:
 fmt:
 	go fmt ./...
 
+fmt-more:
+	gofumpt -l -w .
+
+gci:
+	gci write -s standard -s default -s "prefix(github.com/fatedier/frp/)" ./
+
 vet:
 	go vet ./...
 

Makefile.cross-compiles

@@ -2,7 +2,7 @@ export PATH := $(GOPATH)/bin:$(PATH)
 export GO111MODULE=on
 LDFLAGS := -s -w
 
-os-archs=darwin:amd64 darwin:arm64 freebsd:386 freebsd:amd64 linux:386 linux:amd64 linux:arm linux:arm64 windows:386 windows:amd64 linux:mips64 linux:mips64le linux:mips:softfloat linux:mipsle:softfloat linux:riscv64
+os-archs=darwin:amd64 darwin:arm64 freebsd:386 freebsd:amd64 linux:386 linux:amd64 linux:arm linux:arm64 windows:386 windows:amd64 windows:arm64 linux:mips64 linux:mips64le linux:mips:softfloat linux:mipsle:softfloat linux:riscv64
 
 all: build
 
@@ -23,3 +23,5 @@ app:
 	@mv ./release/frps_windows_386 ./release/frps_windows_386.exe
 	@mv ./release/frpc_windows_amd64 ./release/frpc_windows_amd64.exe
 	@mv ./release/frps_windows_amd64 ./release/frps_windows_amd64.exe
+	@mv ./release/frpc_windows_arm64 ./release/frpc_windows_arm64.exe
+	@mv ./release/frps_windows_arm64 ./release/frps_windows_arm64.exe

README.md

@@ -1,4 +1,3 @@
-
 # frp
 
 [![Build Status](https://circleci.com/gh/fatedier/frp.svg?style=shield)](https://circleci.com/gh/fatedier/frp)
@@ -8,24 +7,18 @@
 
 <h3 align="center">Gold Sponsors</h3>
 <!--gold sponsors start-->
-
 <p align="center">
   <a href="https://workos.com/?utm_campaign=github_repo&utm_medium=referral&utm_content=frp&utm_source=github" target="_blank">
-    <img width="300px" src="https://raw.githubusercontent.com/fatedier/frp/dev/doc/pic/sponsor_workos.png">
+    <img width="350px" src="https://raw.githubusercontent.com/fatedier/frp/dev/doc/pic/sponsor_workos.png">
   </a>
 </p>
-
 <!--gold sponsors end-->
 
-<h3 align="center">Silver Sponsors</h3>
-
-* Sakura Frp - 欢迎点击 "加入我们"
-
 ## What is frp?
 
-frp is a fast reverse proxy to help you expose a local server behind a NAT or firewall to the Internet. As of now, it supports **TCP** and **UDP**, as well as **HTTP** and **HTTPS** protocols, where requests can be forwarded to internal services by domain name.
+frp is a fast reverse proxy that allows you to expose a local server located behind a NAT or firewall to the Internet. It currently supports **TCP** and **UDP**, as well as **HTTP** and **HTTPS** protocols, enabling requests to be forwarded to internal services via domain name.
 
-frp also has a P2P connect mode.
+frp also offers a P2P connect mode.
 
 ## Table of Contents
 
@@ -34,12 +27,12 @@ frp also has a P2P connect mode.
 * [Development Status](#development-status)
 * [Architecture](#architecture)
 * [Example Usage](#example-usage)
-    * [Access your computer in LAN by SSH](#access-your-computer-in-lan-by-ssh)
-    * [Visit your web service in LAN by custom domains](#visit-your-web-service-in-lan-by-custom-domains)
-    * [Forward DNS query request](#forward-dns-query-request)
-    * [Forward Unix domain socket](#forward-unix-domain-socket)
+    * [Access your computer in a LAN network via SSH](#access-your-computer-in-a-lan-network-via-ssh)
+    * [Accessing Internal Web Services with Custom Domains in LAN](#accessing-internal-web-services-with-custom-domains-in-lan)
+    * [Forward DNS query requests](#forward-dns-query-requests)
+    * [Forward Unix Domain Socket](#forward-unix-domain-socket)
     * [Expose a simple HTTP file server](#expose-a-simple-http-file-server)
-    * [Enable HTTPS for local HTTP(S) service](#enable-https-for-local-https-service)
+    * [Enable HTTPS for a local HTTP(S) service](#enable-https-for-a-local-https-service)
     * [Expose your service privately](#expose-your-service-privately)
     * [P2P Mode](#p2p-mode)
 * [Features](#features)
@@ -90,11 +83,11 @@ frp also has a P2P connect mode.
 
 ## Development Status
 
-frp is under development. Try the latest release version in the `master` branch, or use the `dev` branch for the version in development.
+frp is currently under development. You can try the latest release version in the `master` branch, or use the `dev` branch to access the version currently in development.
 
-We are working on v2 version and trying to do some code refactor and improvements. It won't be compatible with v1.
+We are currently working on version 2 and attempting to perform some code refactoring and improvements. However, please note that it will not be compatible with version 1.
 
-We will switch v0 to v1 at the right time and only accept bug fixes and improvements instead of big feature requirements.
+We will transition from version 0 to version 1 at the appropriate time and will only accept bug fixes and improvements, rather than big feature requests.
 
 ## Architecture
 
@@ -102,15 +95,15 @@ We will switch v0 to v1 at the right time and only accept bug fixes and improvem
 
 ## Example Usage
 
-Firstly, download the latest programs from [Release](https://github.com/fatedier/frp/releases) page according to your operating system and architecture.
+To begin, download the latest program for your operating system and architecture from the [Release](https://github.com/fatedier/frp/releases) page.
 
-Put `frps` and `frps.ini` onto your server A with public IP.
+Next, place the `frps` binary and `frps.ini` configuration file on Server A, which has a public IP address.
 
-Put `frpc` and `frpc.ini` onto your server B in LAN (that can't be connected from public Internet).
+Finally, place the `frpc` binary and `frpc.ini` configuration file on Server B, which is located on a LAN that cannot be directly accessed from the public internet.
 
-### Access your computer in LAN by SSH
+### Access your computer in a LAN network via SSH
 
-1. Modify `frps.ini` on server A and set the `bind_port` to be connected to frp clients:
+1. Modify `frps.ini` on server A by setting the `bind_port` for frp clients to connect to:
 
   ```ini
   # frps.ini
@@ -122,7 +115,7 @@ Put `frpc` and `frpc.ini` onto your server B in LAN (that can't be connected fro
 
   `./frps -c ./frps.ini`
 
-3. On server B, modify `frpc.ini` to put in your `frps` server public IP as `server_addr` field:
+3. Modify `frpc.ini` on server B and set the `server_addr` field to the public IP address of your frps server:
 
   ```ini
   # frpc.ini
@@ -137,23 +130,23 @@ Put `frpc` and `frpc.ini` onto your server B in LAN (that can't be connected fro
   remote_port = 6000
   ```
 
-Note that `local_port` (listened on client) and `remote_port` (exposed on server) are for traffic goes in/out the frp system, whereas `server_port` is used between frps.
+Note that the `local_port` (listened on the client) and `remote_port` (exposed on the server) are used for traffic going in and out of the frp system, while the `server_port` is used for communication between frps and frpc.
 
 4. Start `frpc` on server B:
 
   `./frpc -c ./frpc.ini`
 
-5. From another machine, SSH to server B like this (assuming that username is `test`):
+5. To access server B from another machine through server A via SSH (assuming the username is `test`), use the following command:
 
   `ssh -oPort=6000 test@x.x.x.x`
 
-### Visit your web service in LAN by custom domains
+### Accessing Internal Web Services with Custom Domains in LAN
 
-Sometimes we want to expose a local web service behind a NAT network to others for testing with your own domain name and unfortunately we can't resolve a domain name to a local IP.
+Sometimes we need to expose a local web service behind a NAT network to others for testing purposes with our own domain name.
 
-However, we can expose an HTTP(S) service using frp.
+Unfortunately, we cannot resolve a domain name to a local IP. However, we can use frp to expose an HTTP(S) service.
 
-1. Modify `frps.ini`, set the vhost HTTP port to 8080:
+1. Modify `frps.ini` and set the HTTP port for vhost to 8080:
 
   ```ini
   # frps.ini
@@ -166,7 +159,7 @@ However, we can expose an HTTP(S) service using frp.
 
   `./frps -c ./frps.ini`
 
-3. Modify `frpc.ini` and set `server_addr` to the IP address of the remote frps server. The `local_port` is the port of your web service:
+3. Modify `frpc.ini` and set `server_addr` to the IP address of the remote frps server. Specify the `local_port` of your web service:
 
   ```ini
   # frpc.ini
@@ -184,11 +177,11 @@ However, we can expose an HTTP(S) service using frp.
 
   `./frpc -c ./frpc.ini`
 
-5. Resolve A record of `www.example.com` to the public IP of the remote frps server or CNAME record to your origin domain.
+5. Map the A record of `www.example.com` to either the public IP of the remote frps server or a CNAME record pointing to your original domain.
 
-6. Now visit your local web service using url `http://www.example.com:8080`.
+6. Visit your local web service using url `http://www.example.com:8080`.
 
-### Forward DNS query request
+### Forward DNS query requests
 
 1. Modify `frps.ini`:
 
@@ -202,7 +195,7 @@ However, we can expose an HTTP(S) service using frp.
 
   `./frps -c ./frps.ini`
 
-3. Modify `frpc.ini` and set `server_addr` to the IP address of the remote frps server, forward DNS query request to Google Public DNS server `8.8.8.8:53`:
+3. Modify `frpc.ini` and set `server_addr` to the IP address of the remote frps server. Forward DNS query requests to the Google Public DNS server `8.8.8.8:53`:
 
   ```ini
   # frpc.ini
@@ -221,17 +214,17 @@ However, we can expose an HTTP(S) service using frp.
 
   `./frpc -c ./frpc.ini`
 
-5. Test DNS resolution using `dig` command:
+5. Test DNS resolution using the `dig` command:
 
   `dig @x.x.x.x -p 6000 www.google.com`
 
-### Forward Unix domain socket
+### Forward Unix Domain Socket
 
 Expose a Unix domain socket (e.g. the Docker daemon socket) as TCP.
 
-Configure `frps` same as above.
+Configure `frps` as above.
 
-1. Start `frpc` with configuration:
+1. Start `frpc` with the following configuration:
 
   ```ini
   # frpc.ini
@@ -246,17 +239,17 @@ Configure `frps` same as above.
   plugin_unix_path = /var/run/docker.sock
   ```
 
-2. Test: Get Docker version using `curl`:
+2. Test the configuration by getting the docker version using `curl`:
 
   `curl http://x.x.x.x:6000/version`
 
 ### Expose a simple HTTP file server
 
-Browser your files stored in the LAN, from public Internet.
+Expose a simple HTTP file server to access files stored in the LAN from the public Internet.
 
-Configure `frps` same as above.
+Configure `frps` as described above, then:
 
-1. Start `frpc` with configuration:
+1. Start `frpc` with the following configuration:
 
   ```ini
   # frpc.ini
@@ -274,13 +267,13 @@ Configure `frps` same as above.
   plugin_http_passwd = abc
   ```
 
-2. Visit `http://x.x.x.x:6000/static/` from your browser and specify correct user and password to view files in `/tmp/files` on the `frpc` machine.
+2. Visit `http://x.x.x.x:6000/static/` from your browser and specify correct username and password to view files in `/tmp/files` on the `frpc` machine.
 
-### Enable HTTPS for local HTTP(S) service
+### Enable HTTPS for a local HTTP(S) service
 
 You may substitute `https2https` for the plugin, and point the `plugin_local_addr` to a HTTPS endpoint.
 
-1. Start `frpc` with configuration:
+1. Start `frpc` with the following configuration:
 
   ```ini
   # frpc.ini
@@ -304,7 +297,7 @@ You may substitute `https2https` for the plugin, and point the `plugin_local_add
 
 ### Expose your service privately
 
-Some services will be at risk if exposed directly to the public network. With **STCP** (secret TCP) mode, a preshared key is needed to access the service from another client.
+To mitigate risks associated with exposing certain services directly to the public network, STCP (Secret TCP) mode requires a preshared key to be used for access to the service from other clients.
 
 Configure `frps` same as above.
 
@@ -346,24 +339,19 @@ Configure `frps` same as above.
 
 ### P2P Mode
 
-**xtcp** is designed for transmitting large amounts of data directly between clients. A frps server is still needed, as P2P here only refers the actual data transmission.
+**xtcp** is designed to transmit large amounts of data directly between clients. A frps server is still needed, as P2P here only refers to the actual data transmission.
 
-Note it can't penetrate all types of NAT devices. You might want to fallback to **stcp** if **xtcp** doesn't work.
+Note that it may not work with all types of NAT devices. You might want to fallback to stcp if xtcp doesn't work.
 
-1. In `frps.ini` configure a UDP port for xtcp:
-
-  ```ini
-  # frps.ini
-  bind_udp_port = 7001
-  ```
-
-2. Start `frpc` on machine B, expose the SSH port. Note that `remote_port` field is removed:
+1. Start `frpc` on machine B, and expose the SSH port. Note that the `remote_port` field is removed:
 
   ```ini
   # frpc.ini
   [common]
   server_addr = x.x.x.x
   server_port = 7000
+  # set up a new stun server if the default one is not available.
+  # nat_hole_stun_server = xxx
 
   [p2p_ssh]
   type = xtcp
@@ -372,13 +360,15 @@ Note it can't penetrate all types of NAT devices. You might want to fallback to
   local_port = 22
   ```
 
-3. Start another `frpc` (typically on another machine C) with the config to connect to SSH using P2P mode:
+2. Start another `frpc` (typically on another machine C) with the configuration to connect to SSH using P2P mode:
 
   ```ini
   # frpc.ini
   [common]
   server_addr = x.x.x.x
   server_port = 7000
+  # set up a new stun server if the default one is not available.
+  # nat_hole_stun_server = xxx
 
   [p2p_ssh_visitor]
   type = xtcp
@@ -387,9 +377,11 @@ Note it can't penetrate all types of NAT devices. You might want to fallback to
   sk = abcdefg
   bind_addr = 127.0.0.1
   bind_port = 6000
+  # when automatic tunnel persistence is required, set it to true
+  keep_tunnel_open = false
   ```
 
-4. On machine C, connect to SSH on machine B, using this command:
+3. On machine C, connect to SSH on machine B, using this command:
 
   `ssh -oPort=6000 127.0.0.1`
 
@@ -721,6 +713,8 @@ bandwidth_limit = 1MB
 
 Set `bandwidth_limit` in each proxy's configure to enable this feature. Supported units are `MB` and `KB`.
 
+Set `bandwidth_limit_mode` to `client` or `server` to limit bandwidth on the client or server side. Default is `client`.
+
 ### TCP Stream Multiplexing
 
 frp supports tcp stream multiplexing since v0.10.0 like HTTP2 Multiplexing, in which case all logic connections to the same frpc are multiplexed into the same TCP connection.

README_zh.md

@@ -1,6 +1,6 @@
 # frp
 
-[![Build Status](https://travis-ci.org/fatedier/frp.svg?branch=master)](https://travis-ci.org/fatedier/frp)
+[![Build Status](https://circleci.com/gh/fatedier/frp.svg?style=shield)](https://circleci.com/gh/fatedier/frp)
 [![GitHub release](https://img.shields.io/github/tag/fatedier/frp.svg?label=release)](https://github.com/fatedier/frp/releases)
 
 [README](README.md) | [中文文档](README_zh.md)
@@ -9,19 +9,13 @@ frp 是一个专注于内网穿透的高性能的反向代理应用，支持 TCP
 
 <h3 align="center">Gold Sponsors</h3>
 <!--gold sponsors start-->
-
 <p align="center">
   <a href="https://workos.com/?utm_campaign=github_repo&utm_medium=referral&utm_content=frp&utm_source=github" target="_blank">
-    <img width="300px" src="https://raw.githubusercontent.com/fatedier/frp/dev/doc/pic/sponsor_workos.png">
+    <img width="350px" src="https://raw.githubusercontent.com/fatedier/frp/dev/doc/pic/sponsor_workos.png">
   </a>
 </p>
-
 <!--gold sponsors end-->
 
-<h3 align="center">Silver Sponsors</h3>
-
-* Sakura Frp - 欢迎点击 "加入我们"
-
 ## 为什么使用 frp ？
 
 通过在具有公网 IP 的节点上部署 frp 服务端，可以轻松地将内网服务穿透到公网，同时提供诸多专业的功能特性，这包括：

Release.md

@@ -1,4 +1,19 @@
+## Notes
+
+We have thoroughly refactored xtcp in this version to improve its penetration rate and stability.
+
+In this version, different penetration strategies can be attempted by retrying connections multiple times. Once a hole is successfully punched, the strategy will be recorded in the server cache for future reuse. When new users connect, the successfully penetrated tunnel can be reused instead of punching a new hole.
+
+**Due to a significant refactor of xtcp, this version is not compatible with previous versions of xtcp.**
+
+**To use features related to xtcp, both frpc and frps need to be updated to the latest version.**
+
+### New
+
+* The frpc has added the `nathole discover` command for testing the NAT type of the current network.
+* `XTCP` has been refactored, resulting in a significant improvement in the success rate of penetration.
+* When verifying passwords, use `subtle.ConstantTimeCompare` and introduce a certain delay when the password is incorrect.
+
 ### Fix
 
-* Server Plugin send incorrect op name for NewWorkConn.
-* QUIC stream leak.
+* Fix the problem of lagging when opening multiple table entries in the frps dashboard.

assets/frpc/static/index.html

@@ -1 +1,16 @@
-<!doctype html> <html lang=en> <head> <meta charset=utf-8> <title>frp client admin UI</title> <link rel="shortcut icon" href="favicon.ico"></head> <body> <div id=app></div> <script type="text/javascript" src="manifest.js?5d5774096cf5c1b4d5af"></script><script type="text/javascript" src="vendor.js?dc42700731a508d39009"></script></body> </html> 
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8">
+    <title>frp client admin UI</title>
+  <script type="module" crossorigin src="./index-1c7ed8b0.js"></script>
+  <link rel="stylesheet" href="./index-1e2a7ce0.css">
+</head>
+
+<body>
+    <div id="app"></div>
+    
+</body>
+
+</html>

assets/frpc/static/manifest.js

@@ -1 +0,0 @@
-!function(e){function n(r){if(t[r])return t[r].exports;var o=t[r]={i:r,l:!1,exports:{}};return e[r].call(o.exports,o,o.exports,n),o.l=!0,o.exports}var r=window.webpackJsonp;window.webpackJsonp=function(t,c,u){for(var i,a,f,l=0,s=[];l<t.length;l++)a=t[l],o[a]&&s.push(o[a][0]),o[a]=0;for(i in c)Object.prototype.hasOwnProperty.call(c,i)&&(e[i]=c[i]);for(r&&r(t,c,u);s.length;)s.shift()();if(u)for(l=0;l<u.length;l++)f=n(n.s=u[l]);return f};var t={},o={1:0};n.e=function(e){function r(){i.onerror=i.onload=null,clearTimeout(a);var n=o[e];0!==n&&(n&&n[1](new Error("Loading chunk "+e+" failed.")),o[e]=void 0)}var t=o[e];if(0===t)return new Promise(function(e){e()});if(t)return t[2];var c=new Promise(function(n,r){t=o[e]=[n,r]});t[2]=c;var u=document.getElementsByTagName("head")[0],i=document.createElement("script");i.type="text/javascript",i.charset="utf-8",i.async=!0,i.timeout=12e4,n.nc&&i.setAttribute("nonce",n.nc),i.src=n.p+""+e+".js?"+{0:"dc42700731a508d39009"}[e];var a=setTimeout(r,12e4);return i.onerror=i.onload=r,u.appendChild(i),c},n.m=e,n.c=t,n.i=function(e){return e},n.d=function(e,r,t){n.o(e,r)||Object.defineProperty(e,r,{configurable:!1,enumerable:!0,get:t})},n.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return n.d(r,"a",r),r},n.o=function(e,n){return Object.prototype.hasOwnProperty.call(e,n)},n.p="",n.oe=function(e){throw console.error(e),e}}([]);

assets/frps/static/index.html

@@ -1 +1,16 @@
-<!doctype html> <html lang=en> <head> <meta charset=utf-8> <title>frps dashboard</title> <link rel="shortcut icon" href="favicon.ico"></head> <body> <div id=app></div> <script type="text/javascript" src="manifest.js?5d154ba4c6b342d8c0c3"></script><script type="text/javascript" src="vendor.js?ddbd1f69fb6e67be4b78"></script></body> </html> 
+<!DOCTYPE html>
+<html lang="en" class="dark">
+
+<head>
+    <meta charset="utf-8">
+    <title>frps dashboard</title>
+  <script type="module" crossorigin src="./index-93e38bbf.js"></script>
+  <link rel="stylesheet" href="./index-1e0c7400.css">
+</head>
+
+<body>
+    <div id="app"></div>
+    
+</body>
+
+</html>

assets/frps/static/manifest.js

@@ -1 +0,0 @@
-!function(e){function n(r){if(t[r])return t[r].exports;var o=t[r]={i:r,l:!1,exports:{}};return e[r].call(o.exports,o,o.exports,n),o.l=!0,o.exports}var r=window.webpackJsonp;window.webpackJsonp=function(t,u,c){for(var i,a,f,l=0,s=[];l<t.length;l++)a=t[l],o[a]&&s.push(o[a][0]),o[a]=0;for(i in u)Object.prototype.hasOwnProperty.call(u,i)&&(e[i]=u[i]);for(r&&r(t,u,c);s.length;)s.shift()();if(c)for(l=0;l<c.length;l++)f=n(n.s=c[l]);return f};var t={},o={1:0};n.e=function(e){function r(){i.onerror=i.onload=null,clearTimeout(a);var n=o[e];0!==n&&(n&&n[1](new Error("Loading chunk "+e+" failed.")),o[e]=void 0)}var t=o[e];if(0===t)return new Promise(function(e){e()});if(t)return t[2];var u=new Promise(function(n,r){t=o[e]=[n,r]});t[2]=u;var c=document.getElementsByTagName("head")[0],i=document.createElement("script");i.type="text/javascript",i.charset="utf-8",i.async=!0,i.timeout=12e4,n.nc&&i.setAttribute("nonce",n.nc),i.src=n.p+""+e+".js?"+{0:"ddbd1f69fb6e67be4b78"}[e];var a=setTimeout(r,12e4);return i.onerror=i.onload=r,c.appendChild(i),u},n.m=e,n.c=t,n.i=function(e){return e},n.d=function(e,r,t){n.o(e,r)||Object.defineProperty(e,r,{configurable:!1,enumerable:!0,get:t})},n.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return n.d(r,"a",r),r},n.o=function(e,n){return Object.prototype.hasOwnProperty.call(e,n)},n.p="",n.oe=function(e){throw console.error(e),e}}([]);

client/admin.go

@@ -48,7 +48,7 @@ func (svr *Service) RunAdminServer(address string) (err error) {
 
 	subRouter := router.NewRoute().Subrouter()
 	user, passwd := svr.cfg.AdminUser, svr.cfg.AdminPwd
-	subRouter.Use(frpNet.NewHTTPAuthMiddleware(user, passwd).Middleware)
+	subRouter.Use(frpNet.NewHTTPAuthMiddleware(user, passwd).SetAuthFailDelay(200 * time.Millisecond).Middleware)
 
 	// api, see admin_api.go
 	subRouter.HandleFunc("/api/reload", svr.apiReload).Methods("GET")

client/admin_api.go

@@ -25,6 +25,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/samber/lo"
+
 	"github.com/fatedier/frp/client/proxy"
 	"github.com/fatedier/frp/pkg/config"
 	"github.com/fatedier/frp/pkg/util/log"
@@ -70,15 +72,7 @@ func (svr *Service) apiReload(w http.ResponseWriter, r *http.Request) {
 	log.Info("success reload conf")
 }
 
-type StatusResp struct {
-	TCP   []ProxyStatusResp `json:"tcp"`
-	UDP   []ProxyStatusResp `json:"udp"`
-	HTTP  []ProxyStatusResp `json:"http"`
-	HTTPS []ProxyStatusResp `json:"https"`
-	STCP  []ProxyStatusResp `json:"stcp"`
-	XTCP  []ProxyStatusResp `json:"xtcp"`
-	SUDP  []ProxyStatusResp `json:"sudp"`
-}
+type StatusResp map[string][]ProxyStatusResp
 
 type ProxyStatusResp struct {
 	Name       string `json:"name"`
@@ -90,12 +84,6 @@ type ProxyStatusResp struct {
 	RemoteAddr string `json:"remote_addr"`
 }
 
-type ByProxyStatusResp []ProxyStatusResp
-
-func (a ByProxyStatusResp) Len() int           { return len(a) }
-func (a ByProxyStatusResp) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a ByProxyStatusResp) Less(i, j int) bool { return strings.Compare(a[i].Name, a[j].Name) < 0 }
-
 func NewProxyStatusResp(status *proxy.WorkingStatus, serverAddr string) ProxyStatusResp {
 	psr := ProxyStatusResp{
 		Name:   status.Name,
@@ -103,53 +91,17 @@ func NewProxyStatusResp(status *proxy.WorkingStatus, serverAddr string) ProxySta
 		Status: status.Phase,
 		Err:    status.Err,
 	}
-	switch cfg := status.Cfg.(type) {
-	case *config.TCPProxyConf:
-		if cfg.LocalPort != 0 {
-			psr.LocalAddr = net.JoinHostPort(cfg.LocalIP, strconv.Itoa(cfg.LocalPort))
-		}
-		psr.Plugin = cfg.Plugin
-		if status.Err != "" {
-			psr.RemoteAddr = net.JoinHostPort(serverAddr, strconv.Itoa(cfg.RemotePort))
-		} else {
-			psr.RemoteAddr = serverAddr + status.RemoteAddr
-		}
-	case *config.UDPProxyConf:
-		if cfg.LocalPort != 0 {
-			psr.LocalAddr = net.JoinHostPort(cfg.LocalIP, strconv.Itoa(cfg.LocalPort))
-		}
-		if status.Err != "" {
-			psr.RemoteAddr = net.JoinHostPort(serverAddr, strconv.Itoa(cfg.RemotePort))
-		} else {
-			psr.RemoteAddr = serverAddr + status.RemoteAddr
-		}
-	case *config.HTTPProxyConf:
-		if cfg.LocalPort != 0 {
-			psr.LocalAddr = net.JoinHostPort(cfg.LocalIP, strconv.Itoa(cfg.LocalPort))
-		}
-		psr.Plugin = cfg.Plugin
+	baseCfg := status.Cfg.GetBaseInfo()
+	if baseCfg.LocalPort != 0 {
+		psr.LocalAddr = net.JoinHostPort(baseCfg.LocalIP, strconv.Itoa(baseCfg.LocalPort))
+	}
+	psr.Plugin = baseCfg.Plugin
+
+	if status.Err == "" {
 		psr.RemoteAddr = status.RemoteAddr
-	case *config.HTTPSProxyConf:
-		if cfg.LocalPort != 0 {
-			psr.LocalAddr = net.JoinHostPort(cfg.LocalIP, strconv.Itoa(cfg.LocalPort))
+		if lo.Contains([]string{"tcp", "udp"}, status.Type) {
+			psr.RemoteAddr = serverAddr + psr.RemoteAddr
 		}
-		psr.Plugin = cfg.Plugin
-		psr.RemoteAddr = status.RemoteAddr
-	case *config.STCPProxyConf:
-		if cfg.LocalPort != 0 {
-			psr.LocalAddr = net.JoinHostPort(cfg.LocalIP, strconv.Itoa(cfg.LocalPort))
-		}
-		psr.Plugin = cfg.Plugin
-	case *config.XTCPProxyConf:
-		if cfg.LocalPort != 0 {
-			psr.LocalAddr = net.JoinHostPort(cfg.LocalIP, strconv.Itoa(cfg.LocalPort))
-		}
-		psr.Plugin = cfg.Plugin
-	case *config.SUDPProxyConf:
-		if cfg.LocalPort != 0 {
-			psr.LocalAddr = net.JoinHostPort(cfg.LocalIP, strconv.Itoa(cfg.LocalPort))
-		}
-		psr.Plugin = cfg.Plugin
 	}
 	return psr
 }
@@ -158,15 +110,8 @@ func NewProxyStatusResp(status *proxy.WorkingStatus, serverAddr string) ProxySta
 func (svr *Service) apiStatus(w http.ResponseWriter, r *http.Request) {
 	var (
 		buf []byte
-		res StatusResp
+		res StatusResp = make(map[string][]ProxyStatusResp)
 	)
-	res.TCP = make([]ProxyStatusResp, 0)
-	res.UDP = make([]ProxyStatusResp, 0)
-	res.HTTP = make([]ProxyStatusResp, 0)
-	res.HTTPS = make([]ProxyStatusResp, 0)
-	res.STCP = make([]ProxyStatusResp, 0)
-	res.XTCP = make([]ProxyStatusResp, 0)
-	res.SUDP = make([]ProxyStatusResp, 0)
 
 	log.Info("Http request [/api/status]")
 	defer func() {
@@ -177,30 +122,17 @@ func (svr *Service) apiStatus(w http.ResponseWriter, r *http.Request) {
 
 	ps := svr.ctl.pm.GetAllProxyStatus()
 	for _, status := range ps {
-		switch status.Type {
-		case "tcp":
-			res.TCP = append(res.TCP, NewProxyStatusResp(status, svr.cfg.ServerAddr))
-		case "udp":
-			res.UDP = append(res.UDP, NewProxyStatusResp(status, svr.cfg.ServerAddr))
-		case "http":
-			res.HTTP = append(res.HTTP, NewProxyStatusResp(status, svr.cfg.ServerAddr))
-		case "https":
-			res.HTTPS = append(res.HTTPS, NewProxyStatusResp(status, svr.cfg.ServerAddr))
-		case "stcp":
-			res.STCP = append(res.STCP, NewProxyStatusResp(status, svr.cfg.ServerAddr))
-		case "xtcp":
-			res.XTCP = append(res.XTCP, NewProxyStatusResp(status, svr.cfg.ServerAddr))
-		case "sudp":
-			res.SUDP = append(res.SUDP, NewProxyStatusResp(status, svr.cfg.ServerAddr))
-		}
+		res[status.Type] = append(res[status.Type], NewProxyStatusResp(status, svr.cfg.ServerAddr))
+	}
+
+	for _, arrs := range res {
+		if len(arrs) <= 1 {
+			continue
+		}
+		sort.Slice(arrs, func(i, j int) bool {
+			return strings.Compare(arrs[i].Name, arrs[j].Name) < 0
+		})
 	}
-	sort.Sort(ByProxyStatusResp(res.TCP))
-	sort.Sort(ByProxyStatusResp(res.UDP))
-	sort.Sort(ByProxyStatusResp(res.HTTP))
-	sort.Sort(ByProxyStatusResp(res.HTTPS))
-	sort.Sort(ByProxyStatusResp(res.STCP))
-	sort.Sort(ByProxyStatusResp(res.XTCP))
-	sort.Sort(ByProxyStatusResp(res.SUDP))
 }
 
 // GET api/config

client/control.go

@@ -25,14 +25,21 @@ import (
 	"github.com/fatedier/golib/crypto"
 
 	"github.com/fatedier/frp/client/proxy"
+	"github.com/fatedier/frp/client/visitor"
 	"github.com/fatedier/frp/pkg/auth"
 	"github.com/fatedier/frp/pkg/config"
 	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/transport"
 	"github.com/fatedier/frp/pkg/util/xlog"
 )
 
 type Control struct {
-	// uniq id got from frps, attach it in loginMsg
+	// service context
+	ctx context.Context
+	xl  *xlog.Logger
+
+	// Unique ID obtained from frps.
+	// It should be attached to the login message when reconnecting.
 	runID string
 
 	// manage all proxies
@@ -40,7 +47,7 @@ type Control struct {
 	pm      *proxy.Manager
 
 	// manage all visitors
-	vm *VisitorManager
+	vm *visitor.Manager
 
 	// control connection
 	conn net.Conn
@@ -68,16 +75,10 @@ type Control struct {
 	writerShutdown     *shutdown.Shutdown
 	msgHandlerShutdown *shutdown.Shutdown
 
-	// The UDP port that the server is listening on
-	serverUDPPort int
-
-	xl *xlog.Logger
-
-	// service context
-	ctx context.Context
-
 	// sets authentication based on selected method
 	authSetter auth.Setter
+
+	msgTransporter transport.MessageTransporter
 }
 
 func NewControl(
@@ -85,11 +86,12 @@ func NewControl(
 	clientCfg config.ClientCommonConf,
 	pxyCfgs map[string]config.ProxyConf,
 	visitorCfgs map[string]config.VisitorConf,
-	serverUDPPort int,
 	authSetter auth.Setter,
 ) *Control {
 	// new xlog instance
 	ctl := &Control{
+		ctx:                ctx,
+		xl:                 xlog.FromContextSafe(ctx),
 		runID:              runID,
 		conn:               conn,
 		cm:                 cm,
@@ -102,14 +104,12 @@ func NewControl(
 		readerShutdown:     shutdown.New(),
 		writerShutdown:     shutdown.New(),
 		msgHandlerShutdown: shutdown.New(),
-		serverUDPPort:      serverUDPPort,
-		xl:                 xlog.FromContextSafe(ctx),
-		ctx:                ctx,
 		authSetter:         authSetter,
 	}
-	ctl.pm = proxy.NewManager(ctl.ctx, ctl.sendCh, clientCfg, serverUDPPort)
+	ctl.msgTransporter = transport.NewMessageTransporter(ctl.sendCh)
+	ctl.pm = proxy.NewManager(ctl.ctx, clientCfg, ctl.msgTransporter)
 
-	ctl.vm = NewVisitorManager(ctl.ctx, ctl)
+	ctl.vm = visitor.NewManager(ctl.ctx, ctl.clientCfg, ctl.connectServer, ctl.msgTransporter)
 	ctl.vm.Reload(visitorCfgs)
 	return ctl
 }
@@ -173,6 +173,16 @@ func (ctl *Control) HandleNewProxyResp(inMsg *msg.NewProxyResp) {
 	}
 }
 
+func (ctl *Control) HandleNatHoleResp(inMsg *msg.NatHoleResp) {
+	xl := ctl.xl
+
+	// Dispatch the NatHoleResp message to the related proxy.
+	ok := ctl.msgTransporter.DispatchWithType(inMsg, msg.TypeNameNatHoleResp, inMsg.TransactionID)
+	if !ok {
+		xl.Trace("dispatch NatHoleResp message to related proxy error")
+	}
+}
+
 func (ctl *Control) Close() error {
 	return ctl.GracefulClose(0)
 }
@@ -188,7 +198,7 @@ func (ctl *Control) GracefulClose(d time.Duration) error {
 	return nil
 }
 
-// ClosedDoneCh returns a channel which will be closed after all resources are released
+// ClosedDoneCh returns a channel that will be closed after all resources are released
 func (ctl *Control) ClosedDoneCh() <-chan struct{} {
 	return ctl.closedDoneCh
 }
@@ -250,7 +260,7 @@ func (ctl *Control) writer() {
 	}
 }
 
-// msgHandler handles all channel events and do corresponding operations.
+// msgHandler handles all channel events and performs corresponding operations.
 func (ctl *Control) msgHandler() {
 	xl := ctl.xl
 	defer func() {
@@ -307,6 +317,8 @@ func (ctl *Control) msgHandler() {
 				go ctl.HandleReqWorkConn(m)
 			case *msg.NewProxyResp:
 				ctl.HandleNewProxyResp(m)
+			case *msg.NatHoleResp:
+				ctl.HandleNatHoleResp(m)
 			case *msg.Pong:
 				if m.Error != "" {
 					xl.Error("Pong contains error: %s", m.Error)

client/proxy/proxy.go

@@ -24,20 +24,16 @@ import (
 	"sync"
 	"time"
 
-	"github.com/fatedier/golib/errors"
 	frpIo "github.com/fatedier/golib/io"
 	libdial "github.com/fatedier/golib/net/dial"
-	"github.com/fatedier/golib/pool"
-	fmux "github.com/hashicorp/yamux"
 	pp "github.com/pires/go-proxyproto"
 	"golang.org/x/time/rate"
 
 	"github.com/fatedier/frp/pkg/config"
 	"github.com/fatedier/frp/pkg/msg"
 	plugin "github.com/fatedier/frp/pkg/plugin/client"
-	"github.com/fatedier/frp/pkg/proto/udp"
+	"github.com/fatedier/frp/pkg/transport"
 	"github.com/fatedier/frp/pkg/util/limit"
-	frpNet "github.com/fatedier/frp/pkg/util/net"
 	"github.com/fatedier/frp/pkg/util/xlog"
 )
 
@@ -51,19 +47,24 @@ type Proxy interface {
 	Close()
 }
 
-func NewProxy(ctx context.Context, pxyConf config.ProxyConf, clientCfg config.ClientCommonConf, serverUDPPort int) (pxy Proxy) {
+func NewProxy(
+	ctx context.Context,
+	pxyConf config.ProxyConf,
+	clientCfg config.ClientCommonConf,
+	msgTransporter transport.MessageTransporter,
+) (pxy Proxy) {
 	var limiter *rate.Limiter
 	limitBytes := pxyConf.GetBaseInfo().BandwidthLimit.Bytes()
-	if limitBytes > 0 {
+	if limitBytes > 0 && pxyConf.GetBaseInfo().BandwidthLimitMode == config.BandwidthLimitModeClient {
 		limiter = rate.NewLimiter(rate.Limit(float64(limitBytes)), int(limitBytes))
 	}
 
 	baseProxy := BaseProxy{
-		clientCfg:     clientCfg,
-		serverUDPPort: serverUDPPort,
-		limiter:       limiter,
-		xl:            xlog.FromContextSafe(ctx),
-		ctx:           ctx,
+		clientCfg:      clientCfg,
+		limiter:        limiter,
+		msgTransporter: msgTransporter,
+		xl:             xlog.FromContextSafe(ctx),
+		ctx:            ctx,
 	}
 	switch cfg := pxyConf.(type) {
 	case *config.TCPProxyConf:
@@ -112,10 +113,10 @@ func NewProxy(ctx context.Context, pxyConf config.ProxyConf, clientCfg config.Cl
 }
 
 type BaseProxy struct {
-	closed        bool
-	clientCfg     config.ClientCommonConf
-	serverUDPPort int
-	limiter       *rate.Limiter
+	closed         bool
+	clientCfg      config.ClientCommonConf
+	msgTransporter transport.MessageTransporter
+	limiter        *rate.Limiter
 
 	mu  sync.RWMutex
 	xl  *xlog.Logger
@@ -267,462 +268,6 @@ func (pxy *STCPProxy) InWorkConn(conn net.Conn, m *msg.StartWorkConn) {
 		conn, []byte(pxy.clientCfg.Token), m)
 }
 
-// XTCP
-type XTCPProxy struct {
-	*BaseProxy
-
-	cfg         *config.XTCPProxyConf
-	proxyPlugin plugin.Plugin
-}
-
-func (pxy *XTCPProxy) Run() (err error) {
-	if pxy.cfg.Plugin != "" {
-		pxy.proxyPlugin, err = plugin.Create(pxy.cfg.Plugin, pxy.cfg.PluginParams)
-		if err != nil {
-			return
-		}
-	}
-	return
-}
-
-func (pxy *XTCPProxy) Close() {
-	if pxy.proxyPlugin != nil {
-		pxy.proxyPlugin.Close()
-	}
-}
-
-func (pxy *XTCPProxy) InWorkConn(conn net.Conn, m *msg.StartWorkConn) {
-	xl := pxy.xl
-	defer conn.Close()
-	var natHoleSidMsg msg.NatHoleSid
-	err := msg.ReadMsgInto(conn, &natHoleSidMsg)
-	if err != nil {
-		xl.Error("xtcp read from workConn error: %v", err)
-		return
-	}
-
-	natHoleClientMsg := &msg.NatHoleClient{
-		ProxyName: pxy.cfg.ProxyName,
-		Sid:       natHoleSidMsg.Sid,
-	}
-	raddr, _ := net.ResolveUDPAddr("udp",
-		net.JoinHostPort(pxy.clientCfg.ServerAddr, strconv.Itoa(pxy.serverUDPPort)))
-	clientConn, err := net.DialUDP("udp", nil, raddr)
-	if err != nil {
-		xl.Error("dial server udp addr error: %v", err)
-		return
-	}
-	defer clientConn.Close()
-
-	err = msg.WriteMsg(clientConn, natHoleClientMsg)
-	if err != nil {
-		xl.Error("send natHoleClientMsg to server error: %v", err)
-		return
-	}
-
-	// Wait for client address at most 5 seconds.
-	var natHoleRespMsg msg.NatHoleResp
-	_ = clientConn.SetReadDeadline(time.Now().Add(5 * time.Second))
-
-	buf := pool.GetBuf(1024)
-	n, err := clientConn.Read(buf)
-	if err != nil {
-		xl.Error("get natHoleRespMsg error: %v", err)
-		return
-	}
-	err = msg.ReadMsgInto(bytes.NewReader(buf[:n]), &natHoleRespMsg)
-	if err != nil {
-		xl.Error("get natHoleRespMsg error: %v", err)
-		return
-	}
-	_ = clientConn.SetReadDeadline(time.Time{})
-	_ = clientConn.Close()
-
-	if natHoleRespMsg.Error != "" {
-		xl.Error("natHoleRespMsg get error info: %s", natHoleRespMsg.Error)
-		return
-	}
-
-	xl.Trace("get natHoleRespMsg, sid [%s], client address [%s] visitor address [%s]", natHoleRespMsg.Sid, natHoleRespMsg.ClientAddr, natHoleRespMsg.VisitorAddr)
-
-	// Send detect message
-	host, portStr, err := net.SplitHostPort(natHoleRespMsg.VisitorAddr)
-	if err != nil {
-		xl.Error("get NatHoleResp visitor address [%s] error: %v", natHoleRespMsg.VisitorAddr, err)
-	}
-	laddr, _ := net.ResolveUDPAddr("udp", clientConn.LocalAddr().String())
-
-	port, err := strconv.ParseInt(portStr, 10, 64)
-	if err != nil {
-		xl.Error("get natHoleResp visitor address error: %v", natHoleRespMsg.VisitorAddr)
-		return
-	}
-	_ = pxy.sendDetectMsg(host, int(port), laddr, []byte(natHoleRespMsg.Sid))
-	xl.Trace("send all detect msg done")
-
-	if err := msg.WriteMsg(conn, &msg.NatHoleClientDetectOK{}); err != nil {
-		xl.Error("write message error: %v", err)
-		return
-	}
-
-	// Listen for clientConn's address and wait for visitor connection
-	lConn, err := net.ListenUDP("udp", laddr)
-	if err != nil {
-		xl.Error("listen on visitorConn's local address error: %v", err)
-		return
-	}
-	defer lConn.Close()
-
-	_ = lConn.SetReadDeadline(time.Now().Add(8 * time.Second))
-	sidBuf := pool.GetBuf(1024)
-	var uAddr *net.UDPAddr
-	n, uAddr, err = lConn.ReadFromUDP(sidBuf)
-	if err != nil {
-		xl.Warn("get sid from visitor error: %v", err)
-		return
-	}
-	_ = lConn.SetReadDeadline(time.Time{})
-	if string(sidBuf[:n]) != natHoleRespMsg.Sid {
-		xl.Warn("incorrect sid from visitor")
-		return
-	}
-	pool.PutBuf(sidBuf)
-	xl.Info("nat hole connection make success, sid [%s]", natHoleRespMsg.Sid)
-
-	if _, err := lConn.WriteToUDP(sidBuf[:n], uAddr); err != nil {
-		xl.Error("write uaddr error: %v", err)
-		return
-	}
-
-	kcpConn, err := frpNet.NewKCPConnFromUDP(lConn, false, uAddr.String())
-	if err != nil {
-		xl.Error("create kcp connection from udp connection error: %v", err)
-		return
-	}
-
-	fmuxCfg := fmux.DefaultConfig()
-	fmuxCfg.KeepAliveInterval = 5 * time.Second
-	fmuxCfg.LogOutput = io.Discard
-	sess, err := fmux.Server(kcpConn, fmuxCfg)
-	if err != nil {
-		xl.Error("create yamux server from kcp connection error: %v", err)
-		return
-	}
-	defer sess.Close()
-	muxConn, err := sess.Accept()
-	if err != nil {
-		xl.Error("accept for yamux connection error: %v", err)
-		return
-	}
-
-	HandleTCPWorkConnection(pxy.ctx, &pxy.cfg.LocalSvrConf, pxy.proxyPlugin, pxy.cfg.GetBaseInfo(), pxy.limiter,
-		muxConn, []byte(pxy.cfg.Sk), m)
-}
-
-func (pxy *XTCPProxy) sendDetectMsg(addr string, port int, laddr *net.UDPAddr, content []byte) (err error) {
-	daddr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(addr, strconv.Itoa(port)))
-	if err != nil {
-		return err
-	}
-
-	tConn, err := net.DialUDP("udp", laddr, daddr)
-	if err != nil {
-		return err
-	}
-
-	// uConn := ipv4.NewConn(tConn)
-	// uConn.SetTTL(3)
-
-	if _, err := tConn.Write(content); err != nil {
-		return err
-	}
-	return tConn.Close()
-}
-
-// UDP
-type UDPProxy struct {
-	*BaseProxy
-
-	cfg *config.UDPProxyConf
-
-	localAddr *net.UDPAddr
-	readCh    chan *msg.UDPPacket
-
-	// include msg.UDPPacket and msg.Ping
-	sendCh   chan msg.Message
-	workConn net.Conn
-}
-
-func (pxy *UDPProxy) Run() (err error) {
-	pxy.localAddr, err = net.ResolveUDPAddr("udp", net.JoinHostPort(pxy.cfg.LocalIP, strconv.Itoa(pxy.cfg.LocalPort)))
-	if err != nil {
-		return
-	}
-	return
-}
-
-func (pxy *UDPProxy) Close() {
-	pxy.mu.Lock()
-	defer pxy.mu.Unlock()
-
-	if !pxy.closed {
-		pxy.closed = true
-		if pxy.workConn != nil {
-			pxy.workConn.Close()
-		}
-		if pxy.readCh != nil {
-			close(pxy.readCh)
-		}
-		if pxy.sendCh != nil {
-			close(pxy.sendCh)
-		}
-	}
-}
-
-func (pxy *UDPProxy) InWorkConn(conn net.Conn, m *msg.StartWorkConn) {
-	xl := pxy.xl
-	xl.Info("incoming a new work connection for udp proxy, %s", conn.RemoteAddr().String())
-	// close resources releated with old workConn
-	pxy.Close()
-
-	var rwc io.ReadWriteCloser = conn
-	var err error
-	if pxy.limiter != nil {
-		rwc = frpIo.WrapReadWriteCloser(limit.NewReader(conn, pxy.limiter), limit.NewWriter(conn, pxy.limiter), func() error {
-			return conn.Close()
-		})
-	}
-	if pxy.cfg.UseEncryption {
-		rwc, err = frpIo.WithEncryption(rwc, []byte(pxy.clientCfg.Token))
-		if err != nil {
-			conn.Close()
-			xl.Error("create encryption stream error: %v", err)
-			return
-		}
-	}
-	if pxy.cfg.UseCompression {
-		rwc = frpIo.WithCompression(rwc)
-	}
-	conn = frpNet.WrapReadWriteCloserToConn(rwc, conn)
-
-	pxy.mu.Lock()
-	pxy.workConn = conn
-	pxy.readCh = make(chan *msg.UDPPacket, 1024)
-	pxy.sendCh = make(chan msg.Message, 1024)
-	pxy.closed = false
-	pxy.mu.Unlock()
-
-	workConnReaderFn := func(conn net.Conn, readCh chan *msg.UDPPacket) {
-		for {
-			var udpMsg msg.UDPPacket
-			if errRet := msg.ReadMsgInto(conn, &udpMsg); errRet != nil {
-				xl.Warn("read from workConn for udp error: %v", errRet)
-				return
-			}
-			if errRet := errors.PanicToError(func() {
-				xl.Trace("get udp package from workConn: %s", udpMsg.Content)
-				readCh <- &udpMsg
-			}); errRet != nil {
-				xl.Info("reader goroutine for udp work connection closed: %v", errRet)
-				return
-			}
-		}
-	}
-	workConnSenderFn := func(conn net.Conn, sendCh chan msg.Message) {
-		defer func() {
-			xl.Info("writer goroutine for udp work connection closed")
-		}()
-		var errRet error
-		for rawMsg := range sendCh {
-			switch m := rawMsg.(type) {
-			case *msg.UDPPacket:
-				xl.Trace("send udp package to workConn: %s", m.Content)
-			case *msg.Ping:
-				xl.Trace("send ping message to udp workConn")
-			}
-			if errRet = msg.WriteMsg(conn, rawMsg); errRet != nil {
-				xl.Error("udp work write error: %v", errRet)
-				return
-			}
-		}
-	}
-	heartbeatFn := func(sendCh chan msg.Message) {
-		var errRet error
-		for {
-			time.Sleep(time.Duration(30) * time.Second)
-			if errRet = errors.PanicToError(func() {
-				sendCh <- &msg.Ping{}
-			}); errRet != nil {
-				xl.Trace("heartbeat goroutine for udp work connection closed")
-				break
-			}
-		}
-	}
-
-	go workConnSenderFn(pxy.workConn, pxy.sendCh)
-	go workConnReaderFn(pxy.workConn, pxy.readCh)
-	go heartbeatFn(pxy.sendCh)
-	udp.Forwarder(pxy.localAddr, pxy.readCh, pxy.sendCh, int(pxy.clientCfg.UDPPacketSize))
-}
-
-type SUDPProxy struct {
-	*BaseProxy
-
-	cfg *config.SUDPProxyConf
-
-	localAddr *net.UDPAddr
-
-	closeCh chan struct{}
-}
-
-func (pxy *SUDPProxy) Run() (err error) {
-	pxy.localAddr, err = net.ResolveUDPAddr("udp", net.JoinHostPort(pxy.cfg.LocalIP, strconv.Itoa(pxy.cfg.LocalPort)))
-	if err != nil {
-		return
-	}
-	return
-}
-
-func (pxy *SUDPProxy) Close() {
-	pxy.mu.Lock()
-	defer pxy.mu.Unlock()
-	select {
-	case <-pxy.closeCh:
-		return
-	default:
-		close(pxy.closeCh)
-	}
-}
-
-func (pxy *SUDPProxy) InWorkConn(conn net.Conn, m *msg.StartWorkConn) {
-	xl := pxy.xl
-	xl.Info("incoming a new work connection for sudp proxy, %s", conn.RemoteAddr().String())
-
-	var rwc io.ReadWriteCloser = conn
-	var err error
-	if pxy.limiter != nil {
-		rwc = frpIo.WrapReadWriteCloser(limit.NewReader(conn, pxy.limiter), limit.NewWriter(conn, pxy.limiter), func() error {
-			return conn.Close()
-		})
-	}
-	if pxy.cfg.UseEncryption {
-		rwc, err = frpIo.WithEncryption(rwc, []byte(pxy.clientCfg.Token))
-		if err != nil {
-			conn.Close()
-			xl.Error("create encryption stream error: %v", err)
-			return
-		}
-	}
-	if pxy.cfg.UseCompression {
-		rwc = frpIo.WithCompression(rwc)
-	}
-	conn = frpNet.WrapReadWriteCloserToConn(rwc, conn)
-
-	workConn := conn
-	readCh := make(chan *msg.UDPPacket, 1024)
-	sendCh := make(chan msg.Message, 1024)
-	isClose := false
-
-	mu := &sync.Mutex{}
-
-	closeFn := func() {
-		mu.Lock()
-		defer mu.Unlock()
-		if isClose {
-			return
-		}
-
-		isClose = true
-		if workConn != nil {
-			workConn.Close()
-		}
-		close(readCh)
-		close(sendCh)
-	}
-
-	// udp service <- frpc <- frps <- frpc visitor <- user
-	workConnReaderFn := func(conn net.Conn, readCh chan *msg.UDPPacket) {
-		defer closeFn()
-
-		for {
-			// first to check sudp proxy is closed or not
-			select {
-			case <-pxy.closeCh:
-				xl.Trace("frpc sudp proxy is closed")
-				return
-			default:
-			}
-
-			var udpMsg msg.UDPPacket
-			if errRet := msg.ReadMsgInto(conn, &udpMsg); errRet != nil {
-				xl.Warn("read from workConn for sudp error: %v", errRet)
-				return
-			}
-
-			if errRet := errors.PanicToError(func() {
-				readCh <- &udpMsg
-			}); errRet != nil {
-				xl.Warn("reader goroutine for sudp work connection closed: %v", errRet)
-				return
-			}
-		}
-	}
-
-	// udp service -> frpc -> frps -> frpc visitor -> user
-	workConnSenderFn := func(conn net.Conn, sendCh chan msg.Message) {
-		defer func() {
-			closeFn()
-			xl.Info("writer goroutine for sudp work connection closed")
-		}()
-
-		var errRet error
-		for rawMsg := range sendCh {
-			switch m := rawMsg.(type) {
-			case *msg.UDPPacket:
-				xl.Trace("frpc send udp package to frpc visitor, [udp local: %v, remote: %v], [tcp work conn local: %v, remote: %v]",
-					m.LocalAddr.String(), m.RemoteAddr.String(), conn.LocalAddr().String(), conn.RemoteAddr().String())
-			case *msg.Ping:
-				xl.Trace("frpc send ping message to frpc visitor")
-			}
-
-			if errRet = msg.WriteMsg(conn, rawMsg); errRet != nil {
-				xl.Error("sudp work write error: %v", errRet)
-				return
-			}
-		}
-	}
-
-	heartbeatFn := func(sendCh chan msg.Message) {
-		ticker := time.NewTicker(30 * time.Second)
-		defer func() {
-			ticker.Stop()
-			closeFn()
-		}()
-
-		var errRet error
-		for {
-			select {
-			case <-ticker.C:
-				if errRet = errors.PanicToError(func() {
-					sendCh <- &msg.Ping{}
-				}); errRet != nil {
-					xl.Warn("heartbeat goroutine for sudp work connection closed")
-					return
-				}
-			case <-pxy.closeCh:
-				xl.Trace("frpc sudp proxy is closed")
-				return
-			}
-		}
-	}
-
-	go workConnSenderFn(workConn, sendCh)
-	go workConnReaderFn(workConn, readCh)
-	go heartbeatFn(sendCh)
-
-	udp.Forwarder(pxy.localAddr, readCh, sendCh, int(pxy.clientCfg.UDPPacketSize))
-}
-
 // Common handler for tcp work connections.
 func HandleTCPWorkConnection(ctx context.Context, localInfo *config.LocalSvrConf, proxyPlugin plugin.Plugin,
 	baseInfo *config.BaseProxyConf, limiter *rate.Limiter, workConn net.Conn, encKey []byte, m *msg.StartWorkConn,
@@ -815,6 +360,9 @@ func HandleTCPWorkConnection(ctx context.Context, localInfo *config.LocalSvrConf
 		}
 	}
 
-	frpIo.Join(localConn, remote)
+	_, _, errs := frpIo.Join(localConn, remote)
 	xl.Debug("join connections closed")
+	if len(errs) > 0 {
+		xl.Trace("join connections errors: %v", errs)
+	}
 }

client/proxy/proxy_manager.go

@@ -1,3 +1,17 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package proxy
 
 import (
@@ -6,37 +20,36 @@ import (
 	"net"
 	"sync"
 
-	"github.com/fatedier/golib/errors"
-
 	"github.com/fatedier/frp/client/event"
 	"github.com/fatedier/frp/pkg/config"
 	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/transport"
 	"github.com/fatedier/frp/pkg/util/xlog"
 )
 
 type Manager struct {
-	sendCh  chan (msg.Message)
-	proxies map[string]*Wrapper
+	proxies        map[string]*Wrapper
+	msgTransporter transport.MessageTransporter
 
 	closed bool
 	mu     sync.RWMutex
 
 	clientCfg config.ClientCommonConf
 
-	// The UDP port that the server is listening on
-	serverUDPPort int
-
 	ctx context.Context
 }
 
-func NewManager(ctx context.Context, msgSendCh chan (msg.Message), clientCfg config.ClientCommonConf, serverUDPPort int) *Manager {
+func NewManager(
+	ctx context.Context,
+	clientCfg config.ClientCommonConf,
+	msgTransporter transport.MessageTransporter,
+) *Manager {
 	return &Manager{
-		sendCh:        msgSendCh,
-		proxies:       make(map[string]*Wrapper),
-		closed:        false,
-		clientCfg:     clientCfg,
-		serverUDPPort: serverUDPPort,
-		ctx:           ctx,
+		proxies:        make(map[string]*Wrapper),
+		msgTransporter: msgTransporter,
+		closed:         false,
+		clientCfg:      clientCfg,
+		ctx:            ctx,
 	}
 }
 
@@ -86,10 +99,7 @@ func (pm *Manager) HandleEvent(payload interface{}) error {
 		return event.ErrPayloadType
 	}
 
-	err := errors.PanicToError(func() {
-		pm.sendCh <- m
-	})
-	return err
+	return pm.msgTransporter.Send(m)
 }
 
 func (pm *Manager) GetAllProxyStatus() []*WorkingStatus {
@@ -131,7 +141,7 @@ func (pm *Manager) Reload(pxyCfgs map[string]config.ProxyConf) {
 	addPxyNames := make([]string, 0)
 	for name, cfg := range pxyCfgs {
 		if _, ok := pm.proxies[name]; !ok {
-			pxy := NewWrapper(pm.ctx, cfg, pm.clientCfg, pm.HandleEvent, pm.serverUDPPort)
+			pxy := NewWrapper(pm.ctx, cfg, pm.clientCfg, pm.HandleEvent, pm.msgTransporter)
 			pm.proxies[name] = pxy
 			addPxyNames = append(addPxyNames, name)
 

client/proxy/proxy_wrapper.go

@@ -1,3 +1,17 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package proxy
 
 import (
@@ -14,6 +28,7 @@ import (
 	"github.com/fatedier/frp/client/health"
 	"github.com/fatedier/frp/pkg/config"
 	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/transport"
 	"github.com/fatedier/frp/pkg/util/xlog"
 )
 
@@ -56,6 +71,8 @@ type Wrapper struct {
 	// event handler
 	handler event.Handler
 
+	msgTransporter transport.MessageTransporter
+
 	health           uint32
 	lastSendStartMsg time.Time
 	lastStartErr     time.Time
@@ -67,7 +84,13 @@ type Wrapper struct {
 	ctx context.Context
 }
 
-func NewWrapper(ctx context.Context, cfg config.ProxyConf, clientCfg config.ClientCommonConf, eventHandler event.Handler, serverUDPPort int) *Wrapper {
+func NewWrapper(
+	ctx context.Context,
+	cfg config.ProxyConf,
+	clientCfg config.ClientCommonConf,
+	eventHandler event.Handler,
+	msgTransporter transport.MessageTransporter,
+) *Wrapper {
 	baseInfo := cfg.GetBaseInfo()
 	xl := xlog.FromContextSafe(ctx).Spawn().AppendPrefix(baseInfo.ProxyName)
 	pw := &Wrapper{
@@ -80,6 +103,7 @@ func NewWrapper(ctx context.Context, cfg config.ProxyConf, clientCfg config.Clie
 		closeCh:        make(chan struct{}),
 		healthNotifyCh: make(chan struct{}),
 		handler:        eventHandler,
+		msgTransporter: msgTransporter,
 		xl:             xl,
 		ctx:            xlog.NewContext(ctx, xl),
 	}
@@ -92,7 +116,7 @@ func NewWrapper(ctx context.Context, cfg config.ProxyConf, clientCfg config.Clie
 		xl.Trace("enable health check monitor")
 	}
 
-	pw.pxy = NewProxy(pw.ctx, pw.Cfg, clientCfg, serverUDPPort)
+	pw.pxy = NewProxy(pw.ctx, pw.Cfg, clientCfg, pw.msgTransporter)
 	return pw
 }
 

client/proxy/sudp.go

@@ -0,0 +1,190 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proxy
+
+import (
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/fatedier/golib/errors"
+	frpIo "github.com/fatedier/golib/io"
+
+	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/proto/udp"
+	"github.com/fatedier/frp/pkg/util/limit"
+	frpNet "github.com/fatedier/frp/pkg/util/net"
+)
+
+type SUDPProxy struct {
+	*BaseProxy
+
+	cfg *config.SUDPProxyConf
+
+	localAddr *net.UDPAddr
+
+	closeCh chan struct{}
+}
+
+func (pxy *SUDPProxy) Run() (err error) {
+	pxy.localAddr, err = net.ResolveUDPAddr("udp", net.JoinHostPort(pxy.cfg.LocalIP, strconv.Itoa(pxy.cfg.LocalPort)))
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (pxy *SUDPProxy) Close() {
+	pxy.mu.Lock()
+	defer pxy.mu.Unlock()
+	select {
+	case <-pxy.closeCh:
+		return
+	default:
+		close(pxy.closeCh)
+	}
+}
+
+func (pxy *SUDPProxy) InWorkConn(conn net.Conn, m *msg.StartWorkConn) {
+	xl := pxy.xl
+	xl.Info("incoming a new work connection for sudp proxy, %s", conn.RemoteAddr().String())
+
+	var rwc io.ReadWriteCloser = conn
+	var err error
+	if pxy.limiter != nil {
+		rwc = frpIo.WrapReadWriteCloser(limit.NewReader(conn, pxy.limiter), limit.NewWriter(conn, pxy.limiter), func() error {
+			return conn.Close()
+		})
+	}
+	if pxy.cfg.UseEncryption {
+		rwc, err = frpIo.WithEncryption(rwc, []byte(pxy.clientCfg.Token))
+		if err != nil {
+			conn.Close()
+			xl.Error("create encryption stream error: %v", err)
+			return
+		}
+	}
+	if pxy.cfg.UseCompression {
+		rwc = frpIo.WithCompression(rwc)
+	}
+	conn = frpNet.WrapReadWriteCloserToConn(rwc, conn)
+
+	workConn := conn
+	readCh := make(chan *msg.UDPPacket, 1024)
+	sendCh := make(chan msg.Message, 1024)
+	isClose := false
+
+	mu := &sync.Mutex{}
+
+	closeFn := func() {
+		mu.Lock()
+		defer mu.Unlock()
+		if isClose {
+			return
+		}
+
+		isClose = true
+		if workConn != nil {
+			workConn.Close()
+		}
+		close(readCh)
+		close(sendCh)
+	}
+
+	// udp service <- frpc <- frps <- frpc visitor <- user
+	workConnReaderFn := func(conn net.Conn, readCh chan *msg.UDPPacket) {
+		defer closeFn()
+
+		for {
+			// first to check sudp proxy is closed or not
+			select {
+			case <-pxy.closeCh:
+				xl.Trace("frpc sudp proxy is closed")
+				return
+			default:
+			}
+
+			var udpMsg msg.UDPPacket
+			if errRet := msg.ReadMsgInto(conn, &udpMsg); errRet != nil {
+				xl.Warn("read from workConn for sudp error: %v", errRet)
+				return
+			}
+
+			if errRet := errors.PanicToError(func() {
+				readCh <- &udpMsg
+			}); errRet != nil {
+				xl.Warn("reader goroutine for sudp work connection closed: %v", errRet)
+				return
+			}
+		}
+	}
+
+	// udp service -> frpc -> frps -> frpc visitor -> user
+	workConnSenderFn := func(conn net.Conn, sendCh chan msg.Message) {
+		defer func() {
+			closeFn()
+			xl.Info("writer goroutine for sudp work connection closed")
+		}()
+
+		var errRet error
+		for rawMsg := range sendCh {
+			switch m := rawMsg.(type) {
+			case *msg.UDPPacket:
+				xl.Trace("frpc send udp package to frpc visitor, [udp local: %v, remote: %v], [tcp work conn local: %v, remote: %v]",
+					m.LocalAddr.String(), m.RemoteAddr.String(), conn.LocalAddr().String(), conn.RemoteAddr().String())
+			case *msg.Ping:
+				xl.Trace("frpc send ping message to frpc visitor")
+			}
+
+			if errRet = msg.WriteMsg(conn, rawMsg); errRet != nil {
+				xl.Error("sudp work write error: %v", errRet)
+				return
+			}
+		}
+	}
+
+	heartbeatFn := func(sendCh chan msg.Message) {
+		ticker := time.NewTicker(30 * time.Second)
+		defer func() {
+			ticker.Stop()
+			closeFn()
+		}()
+
+		var errRet error
+		for {
+			select {
+			case <-ticker.C:
+				if errRet = errors.PanicToError(func() {
+					sendCh <- &msg.Ping{}
+				}); errRet != nil {
+					xl.Warn("heartbeat goroutine for sudp work connection closed")
+					return
+				}
+			case <-pxy.closeCh:
+				xl.Trace("frpc sudp proxy is closed")
+				return
+			}
+		}
+	}
+
+	go workConnSenderFn(workConn, sendCh)
+	go workConnReaderFn(workConn, readCh)
+	go heartbeatFn(sendCh)
+
+	udp.Forwarder(pxy.localAddr, readCh, sendCh, int(pxy.clientCfg.UDPPacketSize))
+}

client/proxy/udp.go

@@ -0,0 +1,157 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proxy
+
+import (
+	"io"
+	"net"
+	"strconv"
+	"time"
+
+	"github.com/fatedier/golib/errors"
+	frpIo "github.com/fatedier/golib/io"
+
+	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/proto/udp"
+	"github.com/fatedier/frp/pkg/util/limit"
+	frpNet "github.com/fatedier/frp/pkg/util/net"
+)
+
+// UDP
+type UDPProxy struct {
+	*BaseProxy
+
+	cfg *config.UDPProxyConf
+
+	localAddr *net.UDPAddr
+	readCh    chan *msg.UDPPacket
+
+	// include msg.UDPPacket and msg.Ping
+	sendCh   chan msg.Message
+	workConn net.Conn
+}
+
+func (pxy *UDPProxy) Run() (err error) {
+	pxy.localAddr, err = net.ResolveUDPAddr("udp", net.JoinHostPort(pxy.cfg.LocalIP, strconv.Itoa(pxy.cfg.LocalPort)))
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (pxy *UDPProxy) Close() {
+	pxy.mu.Lock()
+	defer pxy.mu.Unlock()
+
+	if !pxy.closed {
+		pxy.closed = true
+		if pxy.workConn != nil {
+			pxy.workConn.Close()
+		}
+		if pxy.readCh != nil {
+			close(pxy.readCh)
+		}
+		if pxy.sendCh != nil {
+			close(pxy.sendCh)
+		}
+	}
+}
+
+func (pxy *UDPProxy) InWorkConn(conn net.Conn, m *msg.StartWorkConn) {
+	xl := pxy.xl
+	xl.Info("incoming a new work connection for udp proxy, %s", conn.RemoteAddr().String())
+	// close resources releated with old workConn
+	pxy.Close()
+
+	var rwc io.ReadWriteCloser = conn
+	var err error
+	if pxy.limiter != nil {
+		rwc = frpIo.WrapReadWriteCloser(limit.NewReader(conn, pxy.limiter), limit.NewWriter(conn, pxy.limiter), func() error {
+			return conn.Close()
+		})
+	}
+	if pxy.cfg.UseEncryption {
+		rwc, err = frpIo.WithEncryption(rwc, []byte(pxy.clientCfg.Token))
+		if err != nil {
+			conn.Close()
+			xl.Error("create encryption stream error: %v", err)
+			return
+		}
+	}
+	if pxy.cfg.UseCompression {
+		rwc = frpIo.WithCompression(rwc)
+	}
+	conn = frpNet.WrapReadWriteCloserToConn(rwc, conn)
+
+	pxy.mu.Lock()
+	pxy.workConn = conn
+	pxy.readCh = make(chan *msg.UDPPacket, 1024)
+	pxy.sendCh = make(chan msg.Message, 1024)
+	pxy.closed = false
+	pxy.mu.Unlock()
+
+	workConnReaderFn := func(conn net.Conn, readCh chan *msg.UDPPacket) {
+		for {
+			var udpMsg msg.UDPPacket
+			if errRet := msg.ReadMsgInto(conn, &udpMsg); errRet != nil {
+				xl.Warn("read from workConn for udp error: %v", errRet)
+				return
+			}
+			if errRet := errors.PanicToError(func() {
+				xl.Trace("get udp package from workConn: %s", udpMsg.Content)
+				readCh <- &udpMsg
+			}); errRet != nil {
+				xl.Info("reader goroutine for udp work connection closed: %v", errRet)
+				return
+			}
+		}
+	}
+	workConnSenderFn := func(conn net.Conn, sendCh chan msg.Message) {
+		defer func() {
+			xl.Info("writer goroutine for udp work connection closed")
+		}()
+		var errRet error
+		for rawMsg := range sendCh {
+			switch m := rawMsg.(type) {
+			case *msg.UDPPacket:
+				xl.Trace("send udp package to workConn: %s", m.Content)
+			case *msg.Ping:
+				xl.Trace("send ping message to udp workConn")
+			}
+			if errRet = msg.WriteMsg(conn, rawMsg); errRet != nil {
+				xl.Error("udp work write error: %v", errRet)
+				return
+			}
+		}
+	}
+	heartbeatFn := func(sendCh chan msg.Message) {
+		var errRet error
+		for {
+			time.Sleep(time.Duration(30) * time.Second)
+			if errRet = errors.PanicToError(func() {
+				sendCh <- &msg.Ping{}
+			}); errRet != nil {
+				xl.Trace("heartbeat goroutine for udp work connection closed")
+				break
+			}
+		}
+	}
+
+	go workConnSenderFn(pxy.workConn, pxy.sendCh)
+	go workConnReaderFn(pxy.workConn, pxy.readCh)
+	go heartbeatFn(pxy.sendCh)
+	udp.Forwarder(pxy.localAddr, pxy.readCh, pxy.sendCh, int(pxy.clientCfg.UDPPacketSize))
+}

client/proxy/xtcp.go

@@ -0,0 +1,200 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package proxy
+
+import (
+	"io"
+	"net"
+	"time"
+
+	fmux "github.com/hashicorp/yamux"
+	"github.com/quic-go/quic-go"
+
+	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/nathole"
+	plugin "github.com/fatedier/frp/pkg/plugin/client"
+	"github.com/fatedier/frp/pkg/transport"
+	frpNet "github.com/fatedier/frp/pkg/util/net"
+)
+
+// XTCP
+type XTCPProxy struct {
+	*BaseProxy
+
+	cfg         *config.XTCPProxyConf
+	proxyPlugin plugin.Plugin
+}
+
+func (pxy *XTCPProxy) Run() (err error) {
+	if pxy.cfg.Plugin != "" {
+		pxy.proxyPlugin, err = plugin.Create(pxy.cfg.Plugin, pxy.cfg.PluginParams)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+func (pxy *XTCPProxy) Close() {
+	if pxy.proxyPlugin != nil {
+		pxy.proxyPlugin.Close()
+	}
+}
+
+func (pxy *XTCPProxy) InWorkConn(conn net.Conn, startWorkConnMsg *msg.StartWorkConn) {
+	xl := pxy.xl
+	defer conn.Close()
+	var natHoleSidMsg msg.NatHoleSid
+	err := msg.ReadMsgInto(conn, &natHoleSidMsg)
+	if err != nil {
+		xl.Error("xtcp read from workConn error: %v", err)
+		return
+	}
+
+	prepareResult, err := nathole.Prepare([]string{pxy.clientCfg.NatHoleSTUNServer})
+	if err != nil {
+		xl.Warn("nathole prepare error: %v", err)
+		return
+	}
+	xl.Info("nathole prepare success, nat type: %s, behavior: %s, addresses: %v, assistedAddresses: %v",
+		prepareResult.NatType, prepareResult.Behavior, prepareResult.Addrs, prepareResult.AssistedAddrs)
+	defer prepareResult.ListenConn.Close()
+
+	// send NatHoleClient msg to server
+	transactionID := nathole.NewTransactionID()
+	natHoleClientMsg := &msg.NatHoleClient{
+		TransactionID: transactionID,
+		ProxyName:     pxy.cfg.ProxyName,
+		Sid:           natHoleSidMsg.Sid,
+		MappedAddrs:   prepareResult.Addrs,
+		AssistedAddrs: prepareResult.AssistedAddrs,
+	}
+
+	natHoleRespMsg, err := nathole.ExchangeInfo(pxy.ctx, pxy.msgTransporter, transactionID, natHoleClientMsg, 5*time.Second)
+	if err != nil {
+		xl.Warn("nathole exchange info error: %v", err)
+		return
+	}
+
+	xl.Info("get natHoleRespMsg, sid [%s], protocol [%s], candidate address %v, assisted address %v, detectBehavior: %+v",
+		natHoleRespMsg.Sid, natHoleRespMsg.Protocol, natHoleRespMsg.CandidateAddrs,
+		natHoleRespMsg.AssistedAddrs, natHoleRespMsg.DetectBehavior)
+
+	listenConn := prepareResult.ListenConn
+	newListenConn, raddr, err := nathole.MakeHole(pxy.ctx, listenConn, natHoleRespMsg, []byte(pxy.cfg.Sk))
+	if err != nil {
+		listenConn.Close()
+		xl.Warn("make hole error: %v", err)
+		_ = pxy.msgTransporter.Send(&msg.NatHoleReport{
+			Sid:     natHoleRespMsg.Sid,
+			Success: false,
+		})
+		return
+	}
+	listenConn = newListenConn
+	xl.Info("establishing nat hole connection successful, sid [%s], remoteAddr [%s]", natHoleRespMsg.Sid, raddr)
+
+	_ = pxy.msgTransporter.Send(&msg.NatHoleReport{
+		Sid:     natHoleRespMsg.Sid,
+		Success: true,
+	})
+
+	if natHoleRespMsg.Protocol == "kcp" {
+		pxy.listenByKCP(listenConn, raddr, startWorkConnMsg)
+		return
+	}
+
+	// default is quic
+	pxy.listenByQUIC(listenConn, raddr, startWorkConnMsg)
+}
+
+func (pxy *XTCPProxy) listenByKCP(listenConn *net.UDPConn, raddr *net.UDPAddr, startWorkConnMsg *msg.StartWorkConn) {
+	xl := pxy.xl
+	listenConn.Close()
+	laddr, _ := net.ResolveUDPAddr("udp", listenConn.LocalAddr().String())
+	lConn, err := net.DialUDP("udp", laddr, raddr)
+	if err != nil {
+		xl.Warn("dial udp error: %v", err)
+		return
+	}
+	defer lConn.Close()
+
+	remote, err := frpNet.NewKCPConnFromUDP(lConn, true, raddr.String())
+	if err != nil {
+		xl.Warn("create kcp connection from udp connection error: %v", err)
+		return
+	}
+
+	fmuxCfg := fmux.DefaultConfig()
+	fmuxCfg.KeepAliveInterval = 10 * time.Second
+	fmuxCfg.MaxStreamWindowSize = 2 * 1024 * 1024
+	fmuxCfg.LogOutput = io.Discard
+	session, err := fmux.Server(remote, fmuxCfg)
+	if err != nil {
+		xl.Error("create mux session error: %v", err)
+		return
+	}
+	defer session.Close()
+
+	for {
+		muxConn, err := session.Accept()
+		if err != nil {
+			xl.Error("accept connection error: %v", err)
+			return
+		}
+		go HandleTCPWorkConnection(pxy.ctx, &pxy.cfg.LocalSvrConf, pxy.proxyPlugin, pxy.cfg.GetBaseInfo(), pxy.limiter,
+			muxConn, []byte(pxy.cfg.Sk), startWorkConnMsg)
+	}
+}
+
+func (pxy *XTCPProxy) listenByQUIC(listenConn *net.UDPConn, _ *net.UDPAddr, startWorkConnMsg *msg.StartWorkConn) {
+	xl := pxy.xl
+	defer listenConn.Close()
+
+	tlsConfig, err := transport.NewServerTLSConfig("", "", "")
+	if err != nil {
+		xl.Warn("create tls config error: %v", err)
+		return
+	}
+	tlsConfig.NextProtos = []string{"frp"}
+	quicListener, err := quic.Listen(listenConn, tlsConfig,
+		&quic.Config{
+			MaxIdleTimeout:     time.Duration(pxy.clientCfg.QUICMaxIdleTimeout) * time.Second,
+			MaxIncomingStreams: int64(pxy.clientCfg.QUICMaxIncomingStreams),
+			KeepAlivePeriod:    time.Duration(pxy.clientCfg.QUICKeepalivePeriod) * time.Second,
+		},
+	)
+	if err != nil {
+		xl.Warn("dial quic error: %v", err)
+		return
+	}
+	// only accept one connection from raddr
+	c, err := quicListener.Accept(pxy.ctx)
+	if err != nil {
+		xl.Error("quic accept connection error: %v", err)
+		return
+	}
+	for {
+		stream, err := c.AcceptStream(pxy.ctx)
+		if err != nil {
+			xl.Debug("quic accept stream error: %v", err)
+			_ = c.CloseWithError(0, "")
+			return
+		}
+		go HandleTCPWorkConnection(pxy.ctx, &pxy.cfg.LocalSvrConf, pxy.proxyPlugin, pxy.cfg.GetBaseInfo(), pxy.limiter,
+			frpNet.QuicStreamToNetConn(stream, c), []byte(pxy.cfg.Sk), startWorkConnMsg)
+	}
+}

client/service.go

@@ -31,7 +31,7 @@ import (
 	"github.com/fatedier/golib/crypto"
 	libdial "github.com/fatedier/golib/net/dial"
 	fmux "github.com/hashicorp/yamux"
-	quic "github.com/lucas-clemente/quic-go"
+	quic "github.com/quic-go/quic-go"
 
 	"github.com/fatedier/frp/assets"
 	"github.com/fatedier/frp/pkg/auth"
@@ -47,6 +47,7 @@ import (
 
 func init() {
 	crypto.DefaultSalt = "frp"
+	// TODO: remove this when we drop support for go1.19
 	rand.Seed(time.Now().UnixNano())
 }
 
@@ -71,9 +72,6 @@ type Service struct {
 	// string if no configuration file was used.
 	cfgFile string
 
-	// This is configured by the login response from frps
-	serverUDPPort int
-
 	exit uint32 // 0 means not exit
 
 	// service context
@@ -114,8 +112,8 @@ func (svr *Service) Run() error {
 	// set custom DNSServer
 	if svr.cfg.DNSServer != "" {
 		dnsAddr := svr.cfg.DNSServer
-		if !strings.Contains(dnsAddr, ":") {
-			dnsAddr += ":53"
+		if _, _, err := net.SplitHostPort(dnsAddr); err != nil {
+			dnsAddr = net.JoinHostPort(dnsAddr, "53")
 		}
 		// Change default dns server for frpc
 		net.DefaultResolver = &net.Resolver{
@@ -140,7 +138,7 @@ func (svr *Service) Run() error {
 			util.RandomSleep(10*time.Second, 0.9, 1.1)
 		} else {
 			// login success
-			ctl := NewControl(svr.ctx, svr.runID, conn, cm, svr.cfg, svr.pxyCfgs, svr.visitorCfgs, svr.serverUDPPort, svr.authSetter)
+			ctl := NewControl(svr.ctx, svr.runID, conn, cm, svr.cfg, svr.pxyCfgs, svr.visitorCfgs, svr.authSetter)
 			ctl.Run()
 			svr.ctlMu.Lock()
 			svr.ctl = ctl
@@ -222,7 +220,7 @@ func (svr *Service) keepControllerWorking() {
 			// reconnect success, init delayTime
 			delayTime = time.Second
 
-			ctl := NewControl(svr.ctx, svr.runID, conn, cm, svr.cfg, svr.pxyCfgs, svr.visitorCfgs, svr.serverUDPPort, svr.authSetter)
+			ctl := NewControl(svr.ctx, svr.runID, conn, cm, svr.cfg, svr.pxyCfgs, svr.visitorCfgs, svr.authSetter)
 			ctl.Run()
 			svr.ctlMu.Lock()
 			if svr.ctl != nil {
@@ -294,8 +292,7 @@ func (svr *Service) login() (conn net.Conn, cm *ConnectionManager, err error) {
 	xl.ResetPrefixes()
 	xl.AppendPrefix(svr.runID)
 
-	svr.serverUDPPort = loginRespMsg.ServerUDPPort
-	xl.Info("login to server success, get run id [%s], server udp port [%d]", loginRespMsg.RunID, loginRespMsg.ServerUDPPort)
+	xl.Info("login to server success, get run id [%s]", loginRespMsg.RunID)
 	return
 }
 

client/visitor.go

@@ -1,568 +0,0 @@
-// Copyright 2017 fatedier, fatedier@gmail.com
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package client
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"io"
-	"net"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/fatedier/golib/errors"
-	frpIo "github.com/fatedier/golib/io"
-	"github.com/fatedier/golib/pool"
-	fmux "github.com/hashicorp/yamux"
-
-	"github.com/fatedier/frp/pkg/config"
-	"github.com/fatedier/frp/pkg/msg"
-	"github.com/fatedier/frp/pkg/proto/udp"
-	frpNet "github.com/fatedier/frp/pkg/util/net"
-	"github.com/fatedier/frp/pkg/util/util"
-	"github.com/fatedier/frp/pkg/util/xlog"
-)
-
-// Visitor is used for forward traffics from local port tot remote service.
-type Visitor interface {
-	Run() error
-	Close()
-}
-
-func NewVisitor(ctx context.Context, ctl *Control, cfg config.VisitorConf) (visitor Visitor) {
-	xl := xlog.FromContextSafe(ctx).Spawn().AppendPrefix(cfg.GetBaseInfo().ProxyName)
-	baseVisitor := BaseVisitor{
-		ctl: ctl,
-		ctx: xlog.NewContext(ctx, xl),
-	}
-	switch cfg := cfg.(type) {
-	case *config.STCPVisitorConf:
-		visitor = &STCPVisitor{
-			BaseVisitor: &baseVisitor,
-			cfg:         cfg,
-		}
-	case *config.XTCPVisitorConf:
-		visitor = &XTCPVisitor{
-			BaseVisitor: &baseVisitor,
-			cfg:         cfg,
-		}
-	case *config.SUDPVisitorConf:
-		visitor = &SUDPVisitor{
-			BaseVisitor:  &baseVisitor,
-			cfg:          cfg,
-			checkCloseCh: make(chan struct{}),
-		}
-	}
-	return
-}
-
-type BaseVisitor struct {
-	ctl *Control
-	l   net.Listener
-
-	mu  sync.RWMutex
-	ctx context.Context
-}
-
-type STCPVisitor struct {
-	*BaseVisitor
-
-	cfg *config.STCPVisitorConf
-}
-
-func (sv *STCPVisitor) Run() (err error) {
-	sv.l, err = net.Listen("tcp", net.JoinHostPort(sv.cfg.BindAddr, strconv.Itoa(sv.cfg.BindPort)))
-	if err != nil {
-		return
-	}
-
-	go sv.worker()
-	return
-}
-
-func (sv *STCPVisitor) Close() {
-	sv.l.Close()
-}
-
-func (sv *STCPVisitor) worker() {
-	xl := xlog.FromContextSafe(sv.ctx)
-	for {
-		conn, err := sv.l.Accept()
-		if err != nil {
-			xl.Warn("stcp local listener closed")
-			return
-		}
-
-		go sv.handleConn(conn)
-	}
-}
-
-func (sv *STCPVisitor) handleConn(userConn net.Conn) {
-	xl := xlog.FromContextSafe(sv.ctx)
-	defer userConn.Close()
-
-	xl.Debug("get a new stcp user connection")
-	visitorConn, err := sv.ctl.connectServer()
-	if err != nil {
-		return
-	}
-	defer visitorConn.Close()
-
-	now := time.Now().Unix()
-	newVisitorConnMsg := &msg.NewVisitorConn{
-		ProxyName:      sv.cfg.ServerName,
-		SignKey:        util.GetAuthKey(sv.cfg.Sk, now),
-		Timestamp:      now,
-		UseEncryption:  sv.cfg.UseEncryption,
-		UseCompression: sv.cfg.UseCompression,
-	}
-	err = msg.WriteMsg(visitorConn, newVisitorConnMsg)
-	if err != nil {
-		xl.Warn("send newVisitorConnMsg to server error: %v", err)
-		return
-	}
-
-	var newVisitorConnRespMsg msg.NewVisitorConnResp
-	_ = visitorConn.SetReadDeadline(time.Now().Add(10 * time.Second))
-	err = msg.ReadMsgInto(visitorConn, &newVisitorConnRespMsg)
-	if err != nil {
-		xl.Warn("get newVisitorConnRespMsg error: %v", err)
-		return
-	}
-	_ = visitorConn.SetReadDeadline(time.Time{})
-
-	if newVisitorConnRespMsg.Error != "" {
-		xl.Warn("start new visitor connection error: %s", newVisitorConnRespMsg.Error)
-		return
-	}
-
-	var remote io.ReadWriteCloser
-	remote = visitorConn
-	if sv.cfg.UseEncryption {
-		remote, err = frpIo.WithEncryption(remote, []byte(sv.cfg.Sk))
-		if err != nil {
-			xl.Error("create encryption stream error: %v", err)
-			return
-		}
-	}
-
-	if sv.cfg.UseCompression {
-		remote = frpIo.WithCompression(remote)
-	}
-
-	frpIo.Join(userConn, remote)
-}
-
-type XTCPVisitor struct {
-	*BaseVisitor
-
-	cfg *config.XTCPVisitorConf
-}
-
-func (sv *XTCPVisitor) Run() (err error) {
-	sv.l, err = net.Listen("tcp", net.JoinHostPort(sv.cfg.BindAddr, strconv.Itoa(sv.cfg.BindPort)))
-	if err != nil {
-		return
-	}
-
-	go sv.worker()
-	return
-}
-
-func (sv *XTCPVisitor) Close() {
-	sv.l.Close()
-}
-
-func (sv *XTCPVisitor) worker() {
-	xl := xlog.FromContextSafe(sv.ctx)
-	for {
-		conn, err := sv.l.Accept()
-		if err != nil {
-			xl.Warn("xtcp local listener closed")
-			return
-		}
-
-		go sv.handleConn(conn)
-	}
-}
-
-func (sv *XTCPVisitor) handleConn(userConn net.Conn) {
-	xl := xlog.FromContextSafe(sv.ctx)
-	defer userConn.Close()
-
-	xl.Debug("get a new xtcp user connection")
-	if sv.ctl.serverUDPPort == 0 {
-		xl.Error("xtcp is not supported by server")
-		return
-	}
-
-	raddr, err := net.ResolveUDPAddr("udp",
-		net.JoinHostPort(sv.ctl.clientCfg.ServerAddr, strconv.Itoa(sv.ctl.serverUDPPort)))
-	if err != nil {
-		xl.Error("resolve server UDP addr error")
-		return
-	}
-
-	visitorConn, err := net.DialUDP("udp", nil, raddr)
-	if err != nil {
-		xl.Warn("dial server udp addr error: %v", err)
-		return
-	}
-	defer visitorConn.Close()
-
-	now := time.Now().Unix()
-	natHoleVisitorMsg := &msg.NatHoleVisitor{
-		ProxyName: sv.cfg.ServerName,
-		SignKey:   util.GetAuthKey(sv.cfg.Sk, now),
-		Timestamp: now,
-	}
-	err = msg.WriteMsg(visitorConn, natHoleVisitorMsg)
-	if err != nil {
-		xl.Warn("send natHoleVisitorMsg to server error: %v", err)
-		return
-	}
-
-	// Wait for client address at most 10 seconds.
-	var natHoleRespMsg msg.NatHoleResp
-	_ = visitorConn.SetReadDeadline(time.Now().Add(10 * time.Second))
-	buf := pool.GetBuf(1024)
-	n, err := visitorConn.Read(buf)
-	if err != nil {
-		xl.Warn("get natHoleRespMsg error: %v", err)
-		return
-	}
-
-	err = msg.ReadMsgInto(bytes.NewReader(buf[:n]), &natHoleRespMsg)
-	if err != nil {
-		xl.Warn("get natHoleRespMsg error: %v", err)
-		return
-	}
-	_ = visitorConn.SetReadDeadline(time.Time{})
-	pool.PutBuf(buf)
-
-	if natHoleRespMsg.Error != "" {
-		xl.Error("natHoleRespMsg get error info: %s", natHoleRespMsg.Error)
-		return
-	}
-
-	xl.Trace("get natHoleRespMsg, sid [%s], client address [%s], visitor address [%s]", natHoleRespMsg.Sid, natHoleRespMsg.ClientAddr, natHoleRespMsg.VisitorAddr)
-
-	// Close visitorConn, so we can use it's local address.
-	visitorConn.Close()
-
-	// send sid message to client
-	laddr, _ := net.ResolveUDPAddr("udp", visitorConn.LocalAddr().String())
-	daddr, err := net.ResolveUDPAddr("udp", natHoleRespMsg.ClientAddr)
-	if err != nil {
-		xl.Error("resolve client udp address error: %v", err)
-		return
-	}
-	lConn, err := net.DialUDP("udp", laddr, daddr)
-	if err != nil {
-		xl.Error("dial client udp address error: %v", err)
-		return
-	}
-	defer lConn.Close()
-
-	if _, err := lConn.Write([]byte(natHoleRespMsg.Sid)); err != nil {
-		xl.Error("write sid error: %v", err)
-		return
-	}
-
-	// read ack sid from client
-	sidBuf := pool.GetBuf(1024)
-	_ = lConn.SetReadDeadline(time.Now().Add(8 * time.Second))
-	n, err = lConn.Read(sidBuf)
-	if err != nil {
-		xl.Warn("get sid from client error: %v", err)
-		return
-	}
-	_ = lConn.SetReadDeadline(time.Time{})
-	if string(sidBuf[:n]) != natHoleRespMsg.Sid {
-		xl.Warn("incorrect sid from client")
-		return
-	}
-	pool.PutBuf(sidBuf)
-
-	xl.Info("nat hole connection make success, sid [%s]", natHoleRespMsg.Sid)
-
-	// wrap kcp connection
-	var remote io.ReadWriteCloser
-	remote, err = frpNet.NewKCPConnFromUDP(lConn, true, natHoleRespMsg.ClientAddr)
-	if err != nil {
-		xl.Error("create kcp connection from udp connection error: %v", err)
-		return
-	}
-
-	fmuxCfg := fmux.DefaultConfig()
-	fmuxCfg.KeepAliveInterval = 5 * time.Second
-	fmuxCfg.LogOutput = io.Discard
-	sess, err := fmux.Client(remote, fmuxCfg)
-	if err != nil {
-		xl.Error("create yamux session error: %v", err)
-		return
-	}
-	defer sess.Close()
-	muxConn, err := sess.Open()
-	if err != nil {
-		xl.Error("open yamux stream error: %v", err)
-		return
-	}
-
-	var muxConnRWCloser io.ReadWriteCloser = muxConn
-	if sv.cfg.UseEncryption {
-		muxConnRWCloser, err = frpIo.WithEncryption(muxConnRWCloser, []byte(sv.cfg.Sk))
-		if err != nil {
-			xl.Error("create encryption stream error: %v", err)
-			return
-		}
-	}
-	if sv.cfg.UseCompression {
-		muxConnRWCloser = frpIo.WithCompression(muxConnRWCloser)
-	}
-
-	frpIo.Join(userConn, muxConnRWCloser)
-	xl.Debug("join connections closed")
-}
-
-type SUDPVisitor struct {
-	*BaseVisitor
-
-	checkCloseCh chan struct{}
-	// udpConn is the listener of udp packet
-	udpConn *net.UDPConn
-	readCh  chan *msg.UDPPacket
-	sendCh  chan *msg.UDPPacket
-
-	cfg *config.SUDPVisitorConf
-}
-
-// SUDP Run start listen a udp port
-func (sv *SUDPVisitor) Run() (err error) {
-	xl := xlog.FromContextSafe(sv.ctx)
-
-	addr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(sv.cfg.BindAddr, strconv.Itoa(sv.cfg.BindPort)))
-	if err != nil {
-		return fmt.Errorf("sudp ResolveUDPAddr error: %v", err)
-	}
-
-	sv.udpConn, err = net.ListenUDP("udp", addr)
-	if err != nil {
-		return fmt.Errorf("listen udp port %s error: %v", addr.String(), err)
-	}
-
-	sv.sendCh = make(chan *msg.UDPPacket, 1024)
-	sv.readCh = make(chan *msg.UDPPacket, 1024)
-
-	xl.Info("sudp start to work, listen on %s", addr)
-
-	go sv.dispatcher()
-	go udp.ForwardUserConn(sv.udpConn, sv.readCh, sv.sendCh, int(sv.ctl.clientCfg.UDPPacketSize))
-
-	return
-}
-
-func (sv *SUDPVisitor) dispatcher() {
-	xl := xlog.FromContextSafe(sv.ctx)
-
-	var (
-		visitorConn net.Conn
-		err         error
-
-		firstPacket *msg.UDPPacket
-	)
-
-	for {
-		select {
-		case firstPacket = <-sv.sendCh:
-			if firstPacket == nil {
-				xl.Info("frpc sudp visitor proxy is closed")
-				return
-			}
-		case <-sv.checkCloseCh:
-			xl.Info("frpc sudp visitor proxy is closed")
-			return
-		}
-
-		visitorConn, err = sv.getNewVisitorConn()
-		if err != nil {
-			xl.Warn("newVisitorConn to frps error: %v, try to reconnect", err)
-			continue
-		}
-
-		// visitorConn always be closed when worker done.
-		sv.worker(visitorConn, firstPacket)
-
-		select {
-		case <-sv.checkCloseCh:
-			return
-		default:
-		}
-	}
-}
-
-func (sv *SUDPVisitor) worker(workConn net.Conn, firstPacket *msg.UDPPacket) {
-	xl := xlog.FromContextSafe(sv.ctx)
-	xl.Debug("starting sudp proxy worker")
-
-	wg := &sync.WaitGroup{}
-	wg.Add(2)
-	closeCh := make(chan struct{})
-
-	// udp service -> frpc -> frps -> frpc visitor -> user
-	workConnReaderFn := func(conn net.Conn) {
-		defer func() {
-			conn.Close()
-			close(closeCh)
-			wg.Done()
-		}()
-
-		for {
-			var (
-				rawMsg msg.Message
-				errRet error
-			)
-
-			// frpc will send heartbeat in workConn to frpc visitor for keeping alive
-			_ = conn.SetReadDeadline(time.Now().Add(60 * time.Second))
-			if rawMsg, errRet = msg.ReadMsg(conn); errRet != nil {
-				xl.Warn("read from workconn for user udp conn error: %v", errRet)
-				return
-			}
-
-			_ = conn.SetReadDeadline(time.Time{})
-			switch m := rawMsg.(type) {
-			case *msg.Ping:
-				xl.Debug("frpc visitor get ping message from frpc")
-				continue
-			case *msg.UDPPacket:
-				if errRet := errors.PanicToError(func() {
-					sv.readCh <- m
-					xl.Trace("frpc visitor get udp packet from workConn: %s", m.Content)
-				}); errRet != nil {
-					xl.Info("reader goroutine for udp work connection closed")
-					return
-				}
-			}
-		}
-	}
-
-	// udp service <- frpc <- frps <- frpc visitor <- user
-	workConnSenderFn := func(conn net.Conn) {
-		defer func() {
-			conn.Close()
-			wg.Done()
-		}()
-
-		var errRet error
-		if firstPacket != nil {
-			if errRet = msg.WriteMsg(conn, firstPacket); errRet != nil {
-				xl.Warn("sender goroutine for udp work connection closed: %v", errRet)
-				return
-			}
-			xl.Trace("send udp package to workConn: %s", firstPacket.Content)
-		}
-
-		for {
-			select {
-			case udpMsg, ok := <-sv.sendCh:
-				if !ok {
-					xl.Info("sender goroutine for udp work connection closed")
-					return
-				}
-
-				if errRet = msg.WriteMsg(conn, udpMsg); errRet != nil {
-					xl.Warn("sender goroutine for udp work connection closed: %v", errRet)
-					return
-				}
-				xl.Trace("send udp package to workConn: %s", udpMsg.Content)
-			case <-closeCh:
-				return
-			}
-		}
-	}
-
-	go workConnReaderFn(workConn)
-	go workConnSenderFn(workConn)
-
-	wg.Wait()
-	xl.Info("sudp worker is closed")
-}
-
-func (sv *SUDPVisitor) getNewVisitorConn() (net.Conn, error) {
-	xl := xlog.FromContextSafe(sv.ctx)
-	visitorConn, err := sv.ctl.connectServer()
-	if err != nil {
-		return nil, fmt.Errorf("frpc connect frps error: %v", err)
-	}
-
-	now := time.Now().Unix()
-	newVisitorConnMsg := &msg.NewVisitorConn{
-		ProxyName:      sv.cfg.ServerName,
-		SignKey:        util.GetAuthKey(sv.cfg.Sk, now),
-		Timestamp:      now,
-		UseEncryption:  sv.cfg.UseEncryption,
-		UseCompression: sv.cfg.UseCompression,
-	}
-	err = msg.WriteMsg(visitorConn, newVisitorConnMsg)
-	if err != nil {
-		return nil, fmt.Errorf("frpc send newVisitorConnMsg to frps error: %v", err)
-	}
-
-	var newVisitorConnRespMsg msg.NewVisitorConnResp
-	_ = visitorConn.SetReadDeadline(time.Now().Add(10 * time.Second))
-	err = msg.ReadMsgInto(visitorConn, &newVisitorConnRespMsg)
-	if err != nil {
-		return nil, fmt.Errorf("frpc read newVisitorConnRespMsg error: %v", err)
-	}
-	_ = visitorConn.SetReadDeadline(time.Time{})
-
-	if newVisitorConnRespMsg.Error != "" {
-		return nil, fmt.Errorf("start new visitor connection error: %s", newVisitorConnRespMsg.Error)
-	}
-
-	var remote io.ReadWriteCloser
-	remote = visitorConn
-	if sv.cfg.UseEncryption {
-		remote, err = frpIo.WithEncryption(remote, []byte(sv.cfg.Sk))
-		if err != nil {
-			xl.Error("create encryption stream error: %v", err)
-			return nil, err
-		}
-	}
-	if sv.cfg.UseCompression {
-		remote = frpIo.WithCompression(remote)
-	}
-	return frpNet.WrapReadWriteCloserToConn(remote, visitorConn), nil
-}
-
-func (sv *SUDPVisitor) Close() {
-	sv.mu.Lock()
-	defer sv.mu.Unlock()
-
-	select {
-	case <-sv.checkCloseCh:
-		return
-	default:
-		close(sv.checkCloseCh)
-	}
-	if sv.udpConn != nil {
-		sv.udpConn.Close()
-	}
-	close(sv.readCh)
-	close(sv.sendCh)
-}

client/visitor/stcp.go

@@ -0,0 +1,118 @@
+// Copyright 2017 fatedier, fatedier@gmail.com
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package visitor
+
+import (
+	"io"
+	"net"
+	"strconv"
+	"time"
+
+	frpIo "github.com/fatedier/golib/io"
+
+	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/util/util"
+	"github.com/fatedier/frp/pkg/util/xlog"
+)
+
+type STCPVisitor struct {
+	*BaseVisitor
+
+	cfg *config.STCPVisitorConf
+}
+
+func (sv *STCPVisitor) Run() (err error) {
+	sv.l, err = net.Listen("tcp", net.JoinHostPort(sv.cfg.BindAddr, strconv.Itoa(sv.cfg.BindPort)))
+	if err != nil {
+		return
+	}
+
+	go sv.worker()
+	return
+}
+
+func (sv *STCPVisitor) Close() {
+	sv.l.Close()
+}
+
+func (sv *STCPVisitor) worker() {
+	xl := xlog.FromContextSafe(sv.ctx)
+	for {
+		conn, err := sv.l.Accept()
+		if err != nil {
+			xl.Warn("stcp local listener closed")
+			return
+		}
+
+		go sv.handleConn(conn)
+	}
+}
+
+func (sv *STCPVisitor) handleConn(userConn net.Conn) {
+	xl := xlog.FromContextSafe(sv.ctx)
+	defer userConn.Close()
+
+	xl.Debug("get a new stcp user connection")
+	visitorConn, err := sv.connectServer()
+	if err != nil {
+		return
+	}
+	defer visitorConn.Close()
+
+	now := time.Now().Unix()
+	newVisitorConnMsg := &msg.NewVisitorConn{
+		ProxyName:      sv.cfg.ServerName,
+		SignKey:        util.GetAuthKey(sv.cfg.Sk, now),
+		Timestamp:      now,
+		UseEncryption:  sv.cfg.UseEncryption,
+		UseCompression: sv.cfg.UseCompression,
+	}
+	err = msg.WriteMsg(visitorConn, newVisitorConnMsg)
+	if err != nil {
+		xl.Warn("send newVisitorConnMsg to server error: %v", err)
+		return
+	}
+
+	var newVisitorConnRespMsg msg.NewVisitorConnResp
+	_ = visitorConn.SetReadDeadline(time.Now().Add(10 * time.Second))
+	err = msg.ReadMsgInto(visitorConn, &newVisitorConnRespMsg)
+	if err != nil {
+		xl.Warn("get newVisitorConnRespMsg error: %v", err)
+		return
+	}
+	_ = visitorConn.SetReadDeadline(time.Time{})
+
+	if newVisitorConnRespMsg.Error != "" {
+		xl.Warn("start new visitor connection error: %s", newVisitorConnRespMsg.Error)
+		return
+	}
+
+	var remote io.ReadWriteCloser
+	remote = visitorConn
+	if sv.cfg.UseEncryption {
+		remote, err = frpIo.WithEncryption(remote, []byte(sv.cfg.Sk))
+		if err != nil {
+			xl.Error("create encryption stream error: %v", err)
+			return
+		}
+	}
+
+	if sv.cfg.UseCompression {
+		remote = frpIo.WithCompression(remote)
+	}
+
+	frpIo.Join(userConn, remote)
+}

client/visitor/sudp.go

@@ -0,0 +1,262 @@
+// Copyright 2017 fatedier, fatedier@gmail.com
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package visitor
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/fatedier/golib/errors"
+	frpIo "github.com/fatedier/golib/io"
+
+	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/proto/udp"
+	frpNet "github.com/fatedier/frp/pkg/util/net"
+	"github.com/fatedier/frp/pkg/util/util"
+	"github.com/fatedier/frp/pkg/util/xlog"
+)
+
+type SUDPVisitor struct {
+	*BaseVisitor
+
+	checkCloseCh chan struct{}
+	// udpConn is the listener of udp packet
+	udpConn *net.UDPConn
+	readCh  chan *msg.UDPPacket
+	sendCh  chan *msg.UDPPacket
+
+	cfg *config.SUDPVisitorConf
+}
+
+// SUDP Run start listen a udp port
+func (sv *SUDPVisitor) Run() (err error) {
+	xl := xlog.FromContextSafe(sv.ctx)
+
+	addr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(sv.cfg.BindAddr, strconv.Itoa(sv.cfg.BindPort)))
+	if err != nil {
+		return fmt.Errorf("sudp ResolveUDPAddr error: %v", err)
+	}
+
+	sv.udpConn, err = net.ListenUDP("udp", addr)
+	if err != nil {
+		return fmt.Errorf("listen udp port %s error: %v", addr.String(), err)
+	}
+
+	sv.sendCh = make(chan *msg.UDPPacket, 1024)
+	sv.readCh = make(chan *msg.UDPPacket, 1024)
+
+	xl.Info("sudp start to work, listen on %s", addr)
+
+	go sv.dispatcher()
+	go udp.ForwardUserConn(sv.udpConn, sv.readCh, sv.sendCh, int(sv.clientCfg.UDPPacketSize))
+
+	return
+}
+
+func (sv *SUDPVisitor) dispatcher() {
+	xl := xlog.FromContextSafe(sv.ctx)
+
+	var (
+		visitorConn net.Conn
+		err         error
+
+		firstPacket *msg.UDPPacket
+	)
+
+	for {
+		select {
+		case firstPacket = <-sv.sendCh:
+			if firstPacket == nil {
+				xl.Info("frpc sudp visitor proxy is closed")
+				return
+			}
+		case <-sv.checkCloseCh:
+			xl.Info("frpc sudp visitor proxy is closed")
+			return
+		}
+
+		visitorConn, err = sv.getNewVisitorConn()
+		if err != nil {
+			xl.Warn("newVisitorConn to frps error: %v, try to reconnect", err)
+			continue
+		}
+
+		// visitorConn always be closed when worker done.
+		sv.worker(visitorConn, firstPacket)
+
+		select {
+		case <-sv.checkCloseCh:
+			return
+		default:
+		}
+	}
+}
+
+func (sv *SUDPVisitor) worker(workConn net.Conn, firstPacket *msg.UDPPacket) {
+	xl := xlog.FromContextSafe(sv.ctx)
+	xl.Debug("starting sudp proxy worker")
+
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+	closeCh := make(chan struct{})
+
+	// udp service -> frpc -> frps -> frpc visitor -> user
+	workConnReaderFn := func(conn net.Conn) {
+		defer func() {
+			conn.Close()
+			close(closeCh)
+			wg.Done()
+		}()
+
+		for {
+			var (
+				rawMsg msg.Message
+				errRet error
+			)
+
+			// frpc will send heartbeat in workConn to frpc visitor for keeping alive
+			_ = conn.SetReadDeadline(time.Now().Add(60 * time.Second))
+			if rawMsg, errRet = msg.ReadMsg(conn); errRet != nil {
+				xl.Warn("read from workconn for user udp conn error: %v", errRet)
+				return
+			}
+
+			_ = conn.SetReadDeadline(time.Time{})
+			switch m := rawMsg.(type) {
+			case *msg.Ping:
+				xl.Debug("frpc visitor get ping message from frpc")
+				continue
+			case *msg.UDPPacket:
+				if errRet := errors.PanicToError(func() {
+					sv.readCh <- m
+					xl.Trace("frpc visitor get udp packet from workConn: %s", m.Content)
+				}); errRet != nil {
+					xl.Info("reader goroutine for udp work connection closed")
+					return
+				}
+			}
+		}
+	}
+
+	// udp service <- frpc <- frps <- frpc visitor <- user
+	workConnSenderFn := func(conn net.Conn) {
+		defer func() {
+			conn.Close()
+			wg.Done()
+		}()
+
+		var errRet error
+		if firstPacket != nil {
+			if errRet = msg.WriteMsg(conn, firstPacket); errRet != nil {
+				xl.Warn("sender goroutine for udp work connection closed: %v", errRet)
+				return
+			}
+			xl.Trace("send udp package to workConn: %s", firstPacket.Content)
+		}
+
+		for {
+			select {
+			case udpMsg, ok := <-sv.sendCh:
+				if !ok {
+					xl.Info("sender goroutine for udp work connection closed")
+					return
+				}
+
+				if errRet = msg.WriteMsg(conn, udpMsg); errRet != nil {
+					xl.Warn("sender goroutine for udp work connection closed: %v", errRet)
+					return
+				}
+				xl.Trace("send udp package to workConn: %s", udpMsg.Content)
+			case <-closeCh:
+				return
+			}
+		}
+	}
+
+	go workConnReaderFn(workConn)
+	go workConnSenderFn(workConn)
+
+	wg.Wait()
+	xl.Info("sudp worker is closed")
+}
+
+func (sv *SUDPVisitor) getNewVisitorConn() (net.Conn, error) {
+	xl := xlog.FromContextSafe(sv.ctx)
+	visitorConn, err := sv.connectServer()
+	if err != nil {
+		return nil, fmt.Errorf("frpc connect frps error: %v", err)
+	}
+
+	now := time.Now().Unix()
+	newVisitorConnMsg := &msg.NewVisitorConn{
+		ProxyName:      sv.cfg.ServerName,
+		SignKey:        util.GetAuthKey(sv.cfg.Sk, now),
+		Timestamp:      now,
+		UseEncryption:  sv.cfg.UseEncryption,
+		UseCompression: sv.cfg.UseCompression,
+	}
+	err = msg.WriteMsg(visitorConn, newVisitorConnMsg)
+	if err != nil {
+		return nil, fmt.Errorf("frpc send newVisitorConnMsg to frps error: %v", err)
+	}
+
+	var newVisitorConnRespMsg msg.NewVisitorConnResp
+	_ = visitorConn.SetReadDeadline(time.Now().Add(10 * time.Second))
+	err = msg.ReadMsgInto(visitorConn, &newVisitorConnRespMsg)
+	if err != nil {
+		return nil, fmt.Errorf("frpc read newVisitorConnRespMsg error: %v", err)
+	}
+	_ = visitorConn.SetReadDeadline(time.Time{})
+
+	if newVisitorConnRespMsg.Error != "" {
+		return nil, fmt.Errorf("start new visitor connection error: %s", newVisitorConnRespMsg.Error)
+	}
+
+	var remote io.ReadWriteCloser
+	remote = visitorConn
+	if sv.cfg.UseEncryption {
+		remote, err = frpIo.WithEncryption(remote, []byte(sv.cfg.Sk))
+		if err != nil {
+			xl.Error("create encryption stream error: %v", err)
+			return nil, err
+		}
+	}
+	if sv.cfg.UseCompression {
+		remote = frpIo.WithCompression(remote)
+	}
+	return frpNet.WrapReadWriteCloserToConn(remote, visitorConn), nil
+}
+
+func (sv *SUDPVisitor) Close() {
+	sv.mu.Lock()
+	defer sv.mu.Unlock()
+
+	select {
+	case <-sv.checkCloseCh:
+		return
+	default:
+		close(sv.checkCloseCh)
+	}
+	if sv.udpConn != nil {
+		sv.udpConn.Close()
+	}
+	close(sv.readCh)
+	close(sv.sendCh)
+}

client/visitor/visitor.go

@@ -0,0 +1,77 @@
+// Copyright 2017 fatedier, fatedier@gmail.com
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package visitor
+
+import (
+	"context"
+	"net"
+	"sync"
+
+	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/transport"
+	"github.com/fatedier/frp/pkg/util/xlog"
+)
+
+// Visitor is used for forward traffics from local port tot remote service.
+type Visitor interface {
+	Run() error
+	Close()
+}
+
+func NewVisitor(
+	ctx context.Context,
+	cfg config.VisitorConf,
+	clientCfg config.ClientCommonConf,
+	connectServer func() (net.Conn, error),
+	msgTransporter transport.MessageTransporter,
+) (visitor Visitor) {
+	xl := xlog.FromContextSafe(ctx).Spawn().AppendPrefix(cfg.GetBaseInfo().ProxyName)
+	baseVisitor := BaseVisitor{
+		clientCfg:      clientCfg,
+		connectServer:  connectServer,
+		msgTransporter: msgTransporter,
+		ctx:            xlog.NewContext(ctx, xl),
+	}
+	switch cfg := cfg.(type) {
+	case *config.STCPVisitorConf:
+		visitor = &STCPVisitor{
+			BaseVisitor: &baseVisitor,
+			cfg:         cfg,
+		}
+	case *config.XTCPVisitorConf:
+		visitor = &XTCPVisitor{
+			BaseVisitor:   &baseVisitor,
+			cfg:           cfg,
+			startTunnelCh: make(chan struct{}),
+		}
+	case *config.SUDPVisitorConf:
+		visitor = &SUDPVisitor{
+			BaseVisitor:  &baseVisitor,
+			cfg:          cfg,
+			checkCloseCh: make(chan struct{}),
+		}
+	}
+	return
+}
+
+type BaseVisitor struct {
+	clientCfg      config.ClientCommonConf
+	connectServer  func() (net.Conn, error)
+	msgTransporter transport.MessageTransporter
+	l              net.Listener
+
+	mu  sync.RWMutex
+	ctx context.Context
+}

client/visitor/visitor_manager.go

@@ -12,22 +12,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package client
+package visitor
 
 import (
 	"context"
+	"net"
 	"sync"
 	"time"
 
 	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/transport"
 	"github.com/fatedier/frp/pkg/util/xlog"
 )
 
-type VisitorManager struct {
-	ctl *Control
-
-	cfgs     map[string]config.VisitorConf
-	visitors map[string]Visitor
+type Manager struct {
+	clientCfg      config.ClientCommonConf
+	connectServer  func() (net.Conn, error)
+	msgTransporter transport.MessageTransporter
+	cfgs           map[string]config.VisitorConf
+	visitors       map[string]Visitor
 
 	checkInterval time.Duration
 
@@ -37,18 +40,25 @@ type VisitorManager struct {
 	stopCh chan struct{}
 }
 
-func NewVisitorManager(ctx context.Context, ctl *Control) *VisitorManager {
-	return &VisitorManager{
-		ctl:           ctl,
-		cfgs:          make(map[string]config.VisitorConf),
-		visitors:      make(map[string]Visitor),
-		checkInterval: 10 * time.Second,
-		ctx:           ctx,
-		stopCh:        make(chan struct{}),
+func NewManager(
+	ctx context.Context,
+	clientCfg config.ClientCommonConf,
+	connectServer func() (net.Conn, error),
+	msgTransporter transport.MessageTransporter,
+) *Manager {
+	return &Manager{
+		clientCfg:      clientCfg,
+		connectServer:  connectServer,
+		msgTransporter: msgTransporter,
+		cfgs:           make(map[string]config.VisitorConf),
+		visitors:       make(map[string]Visitor),
+		checkInterval:  10 * time.Second,
+		ctx:            ctx,
+		stopCh:         make(chan struct{}),
 	}
 }
 
-func (vm *VisitorManager) Run() {
+func (vm *Manager) Run() {
 	xl := xlog.FromContextSafe(vm.ctx)
 
 	ticker := time.NewTicker(vm.checkInterval)
@@ -74,10 +84,10 @@ func (vm *VisitorManager) Run() {
 }
 
 // Hold lock before calling this function.
-func (vm *VisitorManager) startVisitor(cfg config.VisitorConf) (err error) {
+func (vm *Manager) startVisitor(cfg config.VisitorConf) (err error) {
 	xl := xlog.FromContextSafe(vm.ctx)
 	name := cfg.GetBaseInfo().ProxyName
-	visitor := NewVisitor(vm.ctx, vm.ctl, cfg)
+	visitor := NewVisitor(vm.ctx, cfg, vm.clientCfg, vm.connectServer, vm.msgTransporter)
 	err = visitor.Run()
 	if err != nil {
 		xl.Warn("start error: %v", err)
@@ -88,7 +98,7 @@ func (vm *VisitorManager) startVisitor(cfg config.VisitorConf) (err error) {
 	return
 }
 
-func (vm *VisitorManager) Reload(cfgs map[string]config.VisitorConf) {
+func (vm *Manager) Reload(cfgs map[string]config.VisitorConf) {
 	xl := xlog.FromContextSafe(vm.ctx)
 	vm.mu.Lock()
 	defer vm.mu.Unlock()
@@ -129,7 +139,7 @@ func (vm *VisitorManager) Reload(cfgs map[string]config.VisitorConf) {
 	}
 }
 
-func (vm *VisitorManager) Close() {
+func (vm *Manager) Close() {
 	vm.mu.Lock()
 	defer vm.mu.Unlock()
 	for _, v := range vm.visitors {

client/visitor/xtcp.go

@@ -0,0 +1,410 @@
+// Copyright 2017 fatedier, fatedier@gmail.com
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package visitor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	frpIo "github.com/fatedier/golib/io"
+	fmux "github.com/hashicorp/yamux"
+	quic "github.com/quic-go/quic-go"
+	"golang.org/x/time/rate"
+
+	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/nathole"
+	"github.com/fatedier/frp/pkg/transport"
+	frpNet "github.com/fatedier/frp/pkg/util/net"
+	"github.com/fatedier/frp/pkg/util/util"
+	"github.com/fatedier/frp/pkg/util/xlog"
+)
+
+var ErrNoTunnelSession = errors.New("no tunnel session")
+
+type XTCPVisitor struct {
+	*BaseVisitor
+	session       TunnelSession
+	startTunnelCh chan struct{}
+	retryLimiter  *rate.Limiter
+	cancel        context.CancelFunc
+
+	cfg *config.XTCPVisitorConf
+}
+
+func (sv *XTCPVisitor) Run() (err error) {
+	sv.ctx, sv.cancel = context.WithCancel(sv.ctx)
+
+	if sv.cfg.Protocol == "kcp" {
+		sv.session = NewKCPTunnelSession()
+	} else {
+		sv.session = NewQUICTunnelSession(&sv.clientCfg)
+	}
+
+	sv.l, err = net.Listen("tcp", net.JoinHostPort(sv.cfg.BindAddr, strconv.Itoa(sv.cfg.BindPort)))
+	if err != nil {
+		return
+	}
+
+	go sv.worker()
+	go sv.processTunnelStartEvents()
+	if sv.cfg.KeepTunnelOpen {
+		sv.retryLimiter = rate.NewLimiter(rate.Every(time.Hour/time.Duration(sv.cfg.MaxRetriesAnHour)), sv.cfg.MaxRetriesAnHour)
+		go sv.keepTunnelOpenWorker()
+	}
+	return
+}
+
+func (sv *XTCPVisitor) Close() {
+	sv.l.Close()
+	sv.cancel()
+	if sv.session != nil {
+		sv.session.Close()
+	}
+}
+
+func (sv *XTCPVisitor) worker() {
+	xl := xlog.FromContextSafe(sv.ctx)
+	for {
+		conn, err := sv.l.Accept()
+		if err != nil {
+			xl.Warn("xtcp local listener closed")
+			return
+		}
+
+		go sv.handleConn(conn)
+	}
+}
+
+func (sv *XTCPVisitor) processTunnelStartEvents() {
+	for {
+		select {
+		case <-sv.ctx.Done():
+			return
+		case <-sv.startTunnelCh:
+			start := time.Now()
+			sv.makeNatHole()
+			duration := time.Since(start)
+			// avoid too frequently
+			if duration < 10*time.Second {
+				time.Sleep(10*time.Second - duration)
+			}
+		}
+	}
+}
+
+func (sv *XTCPVisitor) keepTunnelOpenWorker() {
+	xl := xlog.FromContextSafe(sv.ctx)
+	ticker := time.NewTicker(time.Duration(sv.cfg.MinRetryInterval) * time.Second)
+	defer ticker.Stop()
+
+	sv.startTunnelCh <- struct{}{}
+	for {
+		select {
+		case <-sv.ctx.Done():
+			return
+		case <-ticker.C:
+			xl.Debug("keepTunnelOpenWorker try to check tunnel...")
+			conn, err := sv.getTunnelConn()
+			if err != nil {
+				xl.Warn("keepTunnelOpenWorker get tunnel connection error: %v", err)
+				_ = sv.retryLimiter.Wait(sv.ctx)
+				continue
+			}
+			xl.Debug("keepTunnelOpenWorker check success")
+			if conn != nil {
+				conn.Close()
+			}
+		}
+	}
+}
+
+func (sv *XTCPVisitor) handleConn(userConn net.Conn) {
+	xl := xlog.FromContextSafe(sv.ctx)
+	defer userConn.Close()
+
+	xl.Debug("get a new xtcp user connection")
+
+	// Open a tunnel connection to the server. If there is already a successful hole-punching connection,
+	// it will be reused. Otherwise, it will block and wait for a successful hole-punching connection until timeout.
+	tunnelConn, err := sv.openTunnel()
+	if err != nil {
+		xl.Error("open tunnel error: %v", err)
+		return
+	}
+
+	var muxConnRWCloser io.ReadWriteCloser = tunnelConn
+	if sv.cfg.UseEncryption {
+		muxConnRWCloser, err = frpIo.WithEncryption(muxConnRWCloser, []byte(sv.cfg.Sk))
+		if err != nil {
+			xl.Error("create encryption stream error: %v", err)
+			return
+		}
+	}
+	if sv.cfg.UseCompression {
+		muxConnRWCloser = frpIo.WithCompression(muxConnRWCloser)
+	}
+
+	_, _, errs := frpIo.Join(userConn, muxConnRWCloser)
+	xl.Debug("join connections closed")
+	if len(errs) > 0 {
+		xl.Trace("join connections errors: %v", errs)
+	}
+}
+
+// openTunnel will open a tunnel connection to the target server.
+func (sv *XTCPVisitor) openTunnel() (conn net.Conn, err error) {
+	xl := xlog.FromContextSafe(sv.ctx)
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+	timeoutC := time.After(20 * time.Second)
+	immediateTrigger := make(chan struct{}, 1)
+	defer close(immediateTrigger)
+	immediateTrigger <- struct{}{}
+
+	for {
+		select {
+		case <-sv.ctx.Done():
+			return nil, sv.ctx.Err()
+		case <-immediateTrigger:
+			conn, err = sv.getTunnelConn()
+		case <-ticker.C:
+			conn, err = sv.getTunnelConn()
+		case <-timeoutC:
+			return nil, fmt.Errorf("open tunnel timeout")
+		}
+
+		if err != nil {
+			if err != ErrNoTunnelSession {
+				xl.Warn("get tunnel connection error: %v", err)
+			}
+			continue
+		}
+		return conn, nil
+	}
+}
+
+func (sv *XTCPVisitor) getTunnelConn() (net.Conn, error) {
+	conn, err := sv.session.OpenConn(sv.ctx)
+	if err == nil {
+		return conn, nil
+	}
+	sv.session.Close()
+
+	select {
+	case sv.startTunnelCh <- struct{}{}:
+	default:
+	}
+	return nil, err
+}
+
+// 0. PreCheck
+// 1. Prepare
+// 2. ExchangeInfo
+// 3. MakeNATHole
+// 4. Create a tunnel session using an underlying UDP connection.
+func (sv *XTCPVisitor) makeNatHole() {
+	xl := xlog.FromContextSafe(sv.ctx)
+	if err := nathole.PreCheck(sv.ctx, sv.msgTransporter, sv.cfg.ServerName, 5*time.Second); err != nil {
+		xl.Warn("nathole precheck error: %v", err)
+		return
+	}
+
+	prepareResult, err := nathole.Prepare([]string{sv.clientCfg.NatHoleSTUNServer})
+	if err != nil {
+		xl.Warn("nathole prepare error: %v", err)
+		return
+	}
+	xl.Info("nathole prepare success, nat type: %s, behavior: %s, addresses: %v, assistedAddresses: %v",
+		prepareResult.NatType, prepareResult.Behavior, prepareResult.Addrs, prepareResult.AssistedAddrs)
+
+	listenConn := prepareResult.ListenConn
+
+	// send NatHoleVisitor to server
+	now := time.Now().Unix()
+	transactionID := nathole.NewTransactionID()
+	natHoleVisitorMsg := &msg.NatHoleVisitor{
+		TransactionID: transactionID,
+		ProxyName:     sv.cfg.ServerName,
+		Protocol:      sv.cfg.Protocol,
+		SignKey:       util.GetAuthKey(sv.cfg.Sk, now),
+		Timestamp:     now,
+		MappedAddrs:   prepareResult.Addrs,
+		AssistedAddrs: prepareResult.AssistedAddrs,
+	}
+
+	natHoleRespMsg, err := nathole.ExchangeInfo(sv.ctx, sv.msgTransporter, transactionID, natHoleVisitorMsg, 5*time.Second)
+	if err != nil {
+		listenConn.Close()
+		xl.Warn("nathole exchange info error: %v", err)
+		return
+	}
+
+	xl.Info("get natHoleRespMsg, sid [%s], protocol [%s], candidate address %v, assisted address %v, detectBehavior: %+v",
+		natHoleRespMsg.Sid, natHoleRespMsg.Protocol, natHoleRespMsg.CandidateAddrs,
+		natHoleRespMsg.AssistedAddrs, natHoleRespMsg.DetectBehavior)
+
+	newListenConn, raddr, err := nathole.MakeHole(sv.ctx, listenConn, natHoleRespMsg, []byte(sv.cfg.Sk))
+	if err != nil {
+		listenConn.Close()
+		xl.Warn("make hole error: %v", err)
+		return
+	}
+	listenConn = newListenConn
+	xl.Info("establishing nat hole connection successful, sid [%s], remoteAddr [%s]", natHoleRespMsg.Sid, raddr)
+
+	if err := sv.session.Init(listenConn, raddr); err != nil {
+		listenConn.Close()
+		xl.Warn("init tunnel session error: %v", err)
+		return
+	}
+}
+
+type TunnelSession interface {
+	Init(listenConn *net.UDPConn, raddr *net.UDPAddr) error
+	OpenConn(context.Context) (net.Conn, error)
+	Close()
+}
+
+type KCPTunnelSession struct {
+	session *fmux.Session
+	lConn   *net.UDPConn
+	mu      sync.RWMutex
+}
+
+func NewKCPTunnelSession() TunnelSession {
+	return &KCPTunnelSession{}
+}
+
+func (ks *KCPTunnelSession) Init(listenConn *net.UDPConn, raddr *net.UDPAddr) error {
+	listenConn.Close()
+	laddr, _ := net.ResolveUDPAddr("udp", listenConn.LocalAddr().String())
+	lConn, err := net.DialUDP("udp", laddr, raddr)
+	if err != nil {
+		return fmt.Errorf("dial udp error: %v", err)
+	}
+	remote, err := frpNet.NewKCPConnFromUDP(lConn, true, raddr.String())
+	if err != nil {
+		return fmt.Errorf("create kcp connection from udp connection error: %v", err)
+	}
+
+	fmuxCfg := fmux.DefaultConfig()
+	fmuxCfg.KeepAliveInterval = 10 * time.Second
+	fmuxCfg.MaxStreamWindowSize = 2 * 1024 * 1024
+	fmuxCfg.LogOutput = io.Discard
+	session, err := fmux.Client(remote, fmuxCfg)
+	if err != nil {
+		remote.Close()
+		return fmt.Errorf("initial client session error: %v", err)
+	}
+	ks.mu.Lock()
+	ks.session = session
+	ks.lConn = lConn
+	ks.mu.Unlock()
+	return nil
+}
+
+func (ks *KCPTunnelSession) OpenConn(ctx context.Context) (net.Conn, error) {
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+	session := ks.session
+	if session == nil {
+		return nil, ErrNoTunnelSession
+	}
+	return session.Open()
+}
+
+func (ks *KCPTunnelSession) Close() {
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+	if ks.session != nil {
+		_ = ks.session.Close()
+		ks.session = nil
+	}
+	if ks.lConn != nil {
+		_ = ks.lConn.Close()
+		ks.lConn = nil
+	}
+}
+
+type QUICTunnelSession struct {
+	session    quic.Connection
+	listenConn *net.UDPConn
+	mu         sync.RWMutex
+
+	clientCfg *config.ClientCommonConf
+}
+
+func NewQUICTunnelSession(clientCfg *config.ClientCommonConf) TunnelSession {
+	return &QUICTunnelSession{
+		clientCfg: clientCfg,
+	}
+}
+
+func (qs *QUICTunnelSession) Init(listenConn *net.UDPConn, raddr *net.UDPAddr) error {
+	tlsConfig, err := transport.NewClientTLSConfig("", "", "", raddr.String())
+	if err != nil {
+		return fmt.Errorf("create tls config error: %v", err)
+	}
+	tlsConfig.NextProtos = []string{"frp"}
+	quicConn, err := quic.Dial(listenConn, raddr, raddr.String(), tlsConfig,
+		&quic.Config{
+			MaxIdleTimeout:     time.Duration(qs.clientCfg.QUICMaxIdleTimeout) * time.Second,
+			MaxIncomingStreams: int64(qs.clientCfg.QUICMaxIncomingStreams),
+			KeepAlivePeriod:    time.Duration(qs.clientCfg.QUICKeepalivePeriod) * time.Second,
+		})
+	if err != nil {
+		return fmt.Errorf("dial quic error: %v", err)
+	}
+	qs.mu.Lock()
+	qs.session = quicConn
+	qs.listenConn = listenConn
+	qs.mu.Unlock()
+	return nil
+}
+
+func (qs *QUICTunnelSession) OpenConn(ctx context.Context) (net.Conn, error) {
+	qs.mu.RLock()
+	defer qs.mu.RUnlock()
+	session := qs.session
+	if session == nil {
+		return nil, ErrNoTunnelSession
+	}
+	stream, err := session.OpenStreamSync(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return frpNet.QuicStreamToNetConn(stream, session), nil
+}
+
+func (qs *QUICTunnelSession) Close() {
+	qs.mu.Lock()
+	defer qs.mu.Unlock()
+	if qs.session != nil {
+		_ = qs.session.CloseWithError(0, "")
+		qs.session = nil
+	}
+	if qs.listenConn != nil {
+		_ = qs.listenConn.Close()
+		qs.listenConn = nil
+	}
+}

cmd/frpc/sub/http.go

@@ -39,6 +39,8 @@ func init() {
 	httpCmd.PersistentFlags().StringVarP(&hostHeaderRewrite, "host_header_rewrite", "", "", "host header rewrite")
 	httpCmd.PersistentFlags().BoolVarP(&useEncryption, "ue", "", false, "use encryption")
 	httpCmd.PersistentFlags().BoolVarP(&useCompression, "uc", "", false, "use compression")
+	httpCmd.PersistentFlags().StringVarP(&bandwidthLimit, "bandwidth_limit", "", "", "bandwidth limit")
+	httpCmd.PersistentFlags().StringVarP(&bandwidthLimitMode, "bandwidth_limit_mode", "", config.BandwidthLimitModeClient, "bandwidth limit mode")
 
 	rootCmd.AddCommand(httpCmd)
 }
@@ -70,6 +72,12 @@ var httpCmd = &cobra.Command{
 		cfg.HostHeaderRewrite = hostHeaderRewrite
 		cfg.UseEncryption = useEncryption
 		cfg.UseCompression = useCompression
+		cfg.BandwidthLimit, err = config.NewBandwidthQuantity(bandwidthLimit)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		cfg.BandwidthLimitMode = bandwidthLimitMode
 
 		err = cfg.CheckForCli()
 		if err != nil {

cmd/frpc/sub/https.go

@@ -35,6 +35,8 @@ func init() {
 	httpsCmd.PersistentFlags().StringVarP(&subDomain, "sd", "", "", "sub domain")
 	httpsCmd.PersistentFlags().BoolVarP(&useEncryption, "ue", "", false, "use encryption")
 	httpsCmd.PersistentFlags().BoolVarP(&useCompression, "uc", "", false, "use compression")
+	httpsCmd.PersistentFlags().StringVarP(&bandwidthLimit, "bandwidth_limit", "", "", "bandwidth limit")
+	httpsCmd.PersistentFlags().StringVarP(&bandwidthLimitMode, "bandwidth_limit_mode", "", config.BandwidthLimitModeClient, "bandwidth limit mode")
 
 	rootCmd.AddCommand(httpsCmd)
 }
@@ -62,6 +64,12 @@ var httpsCmd = &cobra.Command{
 		cfg.SubDomain = subDomain
 		cfg.UseEncryption = useEncryption
 		cfg.UseCompression = useCompression
+		cfg.BandwidthLimit, err = config.NewBandwidthQuantity(bandwidthLimit)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		cfg.BandwidthLimitMode = bandwidthLimitMode
 
 		err = cfg.CheckForCli()
 		if err != nil {

cmd/frpc/sub/nathole.go

@@ -0,0 +1,97 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sub
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/nathole"
+)
+
+var (
+	natHoleSTUNServer string
+	natHoleLocalAddr  string
+)
+
+func init() {
+	RegisterCommonFlags(natholeCmd)
+
+	rootCmd.AddCommand(natholeCmd)
+	natholeCmd.AddCommand(natholeDiscoveryCmd)
+
+	natholeCmd.PersistentFlags().StringVarP(&natHoleSTUNServer, "nat_hole_stun_server", "", "", "STUN server address for nathole")
+	natholeCmd.PersistentFlags().StringVarP(&natHoleLocalAddr, "nat_hole_local_addr", "l", "", "local address to connect STUN server")
+}
+
+var natholeCmd = &cobra.Command{
+	Use:   "nathole",
+	Short: "Actions about nathole",
+}
+
+var natholeDiscoveryCmd = &cobra.Command{
+	Use:   "discover",
+	Short: "Discover nathole information from stun server",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		// ignore error here, because we can use command line pameters
+		cfg, _, _, err := config.ParseClientConfig(cfgFile)
+		if err != nil {
+			cfg = config.GetDefaultClientConf()
+		}
+		if natHoleSTUNServer != "" {
+			cfg.NatHoleSTUNServer = natHoleSTUNServer
+		}
+
+		if err := validateForNatHoleDiscovery(cfg); err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		addrs, localAddr, err := nathole.Discover([]string{cfg.NatHoleSTUNServer}, natHoleLocalAddr)
+		if err != nil {
+			fmt.Println("discover error:", err)
+			os.Exit(1)
+		}
+		if len(addrs) < 2 {
+			fmt.Printf("discover error: can not get enough addresses, need 2, got: %v\n", addrs)
+			os.Exit(1)
+		}
+
+		localIPs, _ := nathole.ListLocalIPsForNatHole(10)
+
+		natFeature, err := nathole.ClassifyNATFeature(addrs, localIPs)
+		if err != nil {
+			fmt.Println("classify nat feature error:", err)
+			os.Exit(1)
+		}
+		fmt.Println("STUN server:", cfg.NatHoleSTUNServer)
+		fmt.Println("Your NAT type is:", natFeature.NatType)
+		fmt.Println("Behavior is:", natFeature.Behavior)
+		fmt.Println("External address is:", addrs)
+		fmt.Println("Local address is:", localAddr.String())
+		fmt.Println("Public Network:", natFeature.PublicNetwork)
+		return nil
+	},
+}
+
+func validateForNatHoleDiscovery(cfg config.ClientCommonConf) error {
+	if cfg.NatHoleSTUNServer == "" {
+		return fmt.Errorf("nat_hole_stun_server can not be empty")
+	}
+	return nil
+}

cmd/frpc/sub/root.go

@@ -53,25 +53,28 @@ var (
 	logFile         string
 	logMaxDays      int
 	disableLogColor bool
+	dnsServer       string
 
-	proxyName         string
-	localIP           string
-	localPort         int
-	remotePort        int
-	useEncryption     bool
-	useCompression    bool
-	customDomains     string
-	subDomain         string
-	httpUser          string
-	httpPwd           string
-	locations         string
-	hostHeaderRewrite string
-	role              string
-	sk                string
-	multiplexer       string
-	serverName        string
-	bindAddr          string
-	bindPort          int
+	proxyName          string
+	localIP            string
+	localPort          int
+	remotePort         int
+	useEncryption      bool
+	useCompression     bool
+	bandwidthLimit     string
+	bandwidthLimitMode string
+	customDomains      string
+	subDomain          string
+	httpUser           string
+	httpPwd            string
+	locations          string
+	hostHeaderRewrite  string
+	role               string
+	sk                 string
+	multiplexer        string
+	serverName         string
+	bindAddr           string
+	bindPort           int
 
 	tlsEnable bool
 )
@@ -92,6 +95,7 @@ func RegisterCommonFlags(cmd *cobra.Command) {
 	cmd.PersistentFlags().IntVarP(&logMaxDays, "log_max_days", "", 3, "log file reversed days")
 	cmd.PersistentFlags().BoolVarP(&disableLogColor, "disable_log_color", "", false, "disable log color in console")
 	cmd.PersistentFlags().BoolVarP(&tlsEnable, "tls_enable", "", false, "enable frpc tls")
+	cmd.PersistentFlags().StringVarP(&dnsServer, "dns_server", "", "", "specify dns server instead of using system default one")
 }
 
 var rootCmd = &cobra.Command{
@@ -106,26 +110,7 @@ var rootCmd = &cobra.Command{
 		// If cfgDir is not empty, run multiple frpc service for each config file in cfgDir.
 		// Note that it's only designed for testing. It's not guaranteed to be stable.
 		if cfgDir != "" {
-			var wg sync.WaitGroup
-			_ = filepath.WalkDir(cfgDir, func(path string, d fs.DirEntry, err error) error {
-				if err != nil {
-					return nil
-				}
-				if d.IsDir() {
-					return nil
-				}
-				wg.Add(1)
-				time.Sleep(time.Millisecond)
-				go func() {
-					defer wg.Done()
-					err := runClient(path)
-					if err != nil {
-						fmt.Printf("frpc service error for config file [%s]\n", path)
-					}
-				}()
-				return nil
-			})
-			wg.Wait()
+			_ = runMultipleClients(cfgDir)
 			return nil
 		}
 
@@ -139,6 +124,27 @@ var rootCmd = &cobra.Command{
 	},
 }
 
+func runMultipleClients(cfgDir string) error {
+	var wg sync.WaitGroup
+	err := filepath.WalkDir(cfgDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return nil
+		}
+		wg.Add(1)
+		time.Sleep(time.Millisecond)
+		go func() {
+			defer wg.Done()
+			err := runClient(path)
+			if err != nil {
+				fmt.Printf("frpc service error for config file [%s]\n", path)
+			}
+		}()
+		return nil
+	})
+	wg.Wait()
+	return err
+}
+
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)
@@ -175,6 +181,7 @@ func parseClientCommonCfgFromCmd() (cfg config.ClientCommonConf, err error) {
 	cfg.LogFile = logFile
 	cfg.LogMaxDays = int64(logMaxDays)
 	cfg.DisableLogColor = disableLogColor
+	cfg.DNSServer = dnsServer
 
 	// Only token authentication is supported in cmd mode
 	cfg.ClientConfig = auth.GetDefaultClientConf()
@@ -216,15 +223,16 @@ func startService(
 		return
 	}
 
-	kcpDoneCh := make(chan struct{})
-	// Capture the exit signal if we use kcp.
-	if cfg.Protocol == "kcp" {
-		go handleSignal(svr, kcpDoneCh)
+	closedDoneCh := make(chan struct{})
+	shouldGracefulClose := cfg.Protocol == "kcp" || cfg.Protocol == "quic"
+	// Capture the exit signal if we use kcp or quic.
+	if shouldGracefulClose {
+		go handleSignal(svr, closedDoneCh)
 	}
 
 	err = svr.Run()
-	if err == nil && cfg.Protocol == "kcp" {
-		<-kcpDoneCh
+	if err == nil && shouldGracefulClose {
+		<-closedDoneCh
 	}
 	return
 }

cmd/frpc/sub/status.go

@@ -81,67 +81,27 @@ func status(clientCfg config.ClientCommonConf) error {
 	if err != nil {
 		return err
 	}
-	res := &client.StatusResp{}
+	res := make(client.StatusResp)
 	err = json.Unmarshal(body, &res)
 	if err != nil {
 		return fmt.Errorf("unmarshal http response error: %s", strings.TrimSpace(string(body)))
 	}
 
 	fmt.Println("Proxy Status...")
-	if len(res.TCP) > 0 {
-		fmt.Println("TCP")
-		tbl := table.New("Name", "Status", "LocalAddr", "Plugin", "RemoteAddr", "Error")
-		for _, ps := range res.TCP {
-			tbl.AddRow(ps.Name, ps.Status, ps.LocalAddr, ps.Plugin, ps.RemoteAddr, ps.Err)
+	types := []string{"tcp", "udp", "tcpmux", "http", "https", "stcp", "sudp", "xtcp"}
+	for _, pxyType := range types {
+		arrs := res[pxyType]
+		if len(arrs) == 0 {
+			continue
 		}
-		tbl.Print()
-		fmt.Println("")
-	}
-	if len(res.UDP) > 0 {
-		fmt.Println("UDP")
-		tbl := table.New("Name", "Status", "LocalAddr", "Plugin", "RemoteAddr", "Error")
-		for _, ps := range res.UDP {
-			tbl.AddRow(ps.Name, ps.Status, ps.LocalAddr, ps.Plugin, ps.RemoteAddr, ps.Err)
-		}
-		tbl.Print()
-		fmt.Println("")
-	}
-	if len(res.HTTP) > 0 {
-		fmt.Println("HTTP")
-		tbl := table.New("Name", "Status", "LocalAddr", "Plugin", "RemoteAddr", "Error")
-		for _, ps := range res.HTTP {
-			tbl.AddRow(ps.Name, ps.Status, ps.LocalAddr, ps.Plugin, ps.RemoteAddr, ps.Err)
-		}
-		tbl.Print()
-		fmt.Println("")
-	}
-	if len(res.HTTPS) > 0 {
-		fmt.Println("HTTPS")
-		tbl := table.New("Name", "Status", "LocalAddr", "Plugin", "RemoteAddr", "Error")
-		for _, ps := range res.HTTPS {
-			tbl.AddRow(ps.Name, ps.Status, ps.LocalAddr, ps.Plugin, ps.RemoteAddr, ps.Err)
-		}
-		tbl.Print()
-		fmt.Println("")
-	}
-	if len(res.STCP) > 0 {
-		fmt.Println("STCP")
-		tbl := table.New("Name", "Status", "LocalAddr", "Plugin", "RemoteAddr", "Error")
-		for _, ps := range res.STCP {
-			tbl.AddRow(ps.Name, ps.Status, ps.LocalAddr, ps.Plugin, ps.RemoteAddr, ps.Err)
-		}
-		tbl.Print()
-		fmt.Println("")
-	}
-	if len(res.XTCP) > 0 {
-		fmt.Println("XTCP")
-		tbl := table.New("Name", "Status", "LocalAddr", "Plugin", "RemoteAddr", "Error")
-		for _, ps := range res.XTCP {
-			tbl.AddRow(ps.Name, ps.Status, ps.LocalAddr, ps.Plugin, ps.RemoteAddr, ps.Err)
-		}
-		tbl.Print()
-		fmt.Println("")
-	}
 
+		fmt.Println(strings.ToUpper(pxyType))
+		tbl := table.New("Name", "Status", "LocalAddr", "Plugin", "RemoteAddr", "Error")
+		for _, ps := range arrs {
+			tbl.AddRow(ps.Name, ps.Status, ps.LocalAddr, ps.Plugin, ps.RemoteAddr, ps.Err)
+		}
+		tbl.Print()
+		fmt.Println("")
+	}
 	return nil
 }

cmd/frpc/sub/stcp.go

@@ -37,6 +37,8 @@ func init() {
 	stcpCmd.PersistentFlags().IntVarP(&bindPort, "bind_port", "", 0, "bind port")
 	stcpCmd.PersistentFlags().BoolVarP(&useEncryption, "ue", "", false, "use encryption")
 	stcpCmd.PersistentFlags().BoolVarP(&useCompression, "uc", "", false, "use compression")
+	stcpCmd.PersistentFlags().StringVarP(&bandwidthLimit, "bandwidth_limit", "", "", "bandwidth limit")
+	stcpCmd.PersistentFlags().StringVarP(&bandwidthLimitMode, "bandwidth_limit_mode", "", config.BandwidthLimitModeClient, "bandwidth limit mode")
 
 	rootCmd.AddCommand(stcpCmd)
 }
@@ -70,6 +72,12 @@ var stcpCmd = &cobra.Command{
 			cfg.Sk = sk
 			cfg.LocalIP = localIP
 			cfg.LocalPort = localPort
+			cfg.BandwidthLimit, err = config.NewBandwidthQuantity(bandwidthLimit)
+			if err != nil {
+				fmt.Println(err)
+				os.Exit(1)
+			}
+			cfg.BandwidthLimitMode = bandwidthLimitMode
 			err = cfg.CheckForCli()
 			if err != nil {
 				fmt.Println(err)

cmd/frpc/sub/sudp.go

@@ -37,6 +37,8 @@ func init() {
 	sudpCmd.PersistentFlags().IntVarP(&bindPort, "bind_port", "", 0, "bind port")
 	sudpCmd.PersistentFlags().BoolVarP(&useEncryption, "ue", "", false, "use encryption")
 	sudpCmd.PersistentFlags().BoolVarP(&useCompression, "uc", "", false, "use compression")
+	sudpCmd.PersistentFlags().StringVarP(&bandwidthLimit, "bandwidth_limit", "", "", "bandwidth limit")
+	sudpCmd.PersistentFlags().StringVarP(&bandwidthLimitMode, "bandwidth_limit_mode", "", config.BandwidthLimitModeClient, "bandwidth limit mode")
 
 	rootCmd.AddCommand(sudpCmd)
 }
@@ -70,6 +72,12 @@ var sudpCmd = &cobra.Command{
 			cfg.Sk = sk
 			cfg.LocalIP = localIP
 			cfg.LocalPort = localPort
+			cfg.BandwidthLimit, err = config.NewBandwidthQuantity(bandwidthLimit)
+			if err != nil {
+				fmt.Println(err)
+				os.Exit(1)
+			}
+			cfg.BandwidthLimitMode = bandwidthLimitMode
 			err = cfg.CheckForCli()
 			if err != nil {
 				fmt.Println(err)

cmd/frpc/sub/tcp.go

@@ -33,6 +33,8 @@ func init() {
 	tcpCmd.PersistentFlags().IntVarP(&remotePort, "remote_port", "r", 0, "remote port")
 	tcpCmd.PersistentFlags().BoolVarP(&useEncryption, "ue", "", false, "use encryption")
 	tcpCmd.PersistentFlags().BoolVarP(&useCompression, "uc", "", false, "use compression")
+	tcpCmd.PersistentFlags().StringVarP(&bandwidthLimit, "bandwidth_limit", "", "", "bandwidth limit")
+	tcpCmd.PersistentFlags().StringVarP(&bandwidthLimitMode, "bandwidth_limit_mode", "", config.BandwidthLimitModeClient, "bandwidth limit mode")
 
 	rootCmd.AddCommand(tcpCmd)
 }
@@ -59,6 +61,12 @@ var tcpCmd = &cobra.Command{
 		cfg.RemotePort = remotePort
 		cfg.UseEncryption = useEncryption
 		cfg.UseCompression = useCompression
+		cfg.BandwidthLimit, err = config.NewBandwidthQuantity(bandwidthLimit)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		cfg.BandwidthLimitMode = bandwidthLimitMode
 
 		err = cfg.CheckForCli()
 		if err != nil {

cmd/frpc/sub/tcpmux.go

@@ -36,6 +36,8 @@ func init() {
 	tcpMuxCmd.PersistentFlags().StringVarP(&multiplexer, "mux", "", "", "multiplexer")
 	tcpMuxCmd.PersistentFlags().BoolVarP(&useEncryption, "ue", "", false, "use encryption")
 	tcpMuxCmd.PersistentFlags().BoolVarP(&useCompression, "uc", "", false, "use compression")
+	tcpMuxCmd.PersistentFlags().StringVarP(&bandwidthLimit, "bandwidth_limit", "", "", "bandwidth limit")
+	tcpMuxCmd.PersistentFlags().StringVarP(&bandwidthLimitMode, "bandwidth_limit_mode", "", config.BandwidthLimitModeClient, "bandwidth limit mode")
 
 	rootCmd.AddCommand(tcpMuxCmd)
 }
@@ -64,6 +66,12 @@ var tcpMuxCmd = &cobra.Command{
 		cfg.Multiplexer = multiplexer
 		cfg.UseEncryption = useEncryption
 		cfg.UseCompression = useCompression
+		cfg.BandwidthLimit, err = config.NewBandwidthQuantity(bandwidthLimit)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		cfg.BandwidthLimitMode = bandwidthLimitMode
 
 		err = cfg.CheckForCli()
 		if err != nil {

cmd/frpc/sub/udp.go

@@ -33,6 +33,8 @@ func init() {
 	udpCmd.PersistentFlags().IntVarP(&remotePort, "remote_port", "r", 0, "remote port")
 	udpCmd.PersistentFlags().BoolVarP(&useEncryption, "ue", "", false, "use encryption")
 	udpCmd.PersistentFlags().BoolVarP(&useCompression, "uc", "", false, "use compression")
+	udpCmd.PersistentFlags().StringVarP(&bandwidthLimit, "bandwidth_limit", "", "", "bandwidth limit")
+	udpCmd.PersistentFlags().StringVarP(&bandwidthLimitMode, "bandwidth_limit_mode", "", config.BandwidthLimitModeClient, "bandwidth limit mode")
 
 	rootCmd.AddCommand(udpCmd)
 }
@@ -59,6 +61,12 @@ var udpCmd = &cobra.Command{
 		cfg.RemotePort = remotePort
 		cfg.UseEncryption = useEncryption
 		cfg.UseCompression = useCompression
+		cfg.BandwidthLimit, err = config.NewBandwidthQuantity(bandwidthLimit)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+		cfg.BandwidthLimitMode = bandwidthLimitMode
 
 		err = cfg.CheckForCli()
 		if err != nil {

cmd/frpc/sub/xtcp.go

@@ -37,6 +37,8 @@ func init() {
 	xtcpCmd.PersistentFlags().IntVarP(&bindPort, "bind_port", "", 0, "bind port")
 	xtcpCmd.PersistentFlags().BoolVarP(&useEncryption, "ue", "", false, "use encryption")
 	xtcpCmd.PersistentFlags().BoolVarP(&useCompression, "uc", "", false, "use compression")
+	xtcpCmd.PersistentFlags().StringVarP(&bandwidthLimit, "bandwidth_limit", "", "", "bandwidth limit")
+	xtcpCmd.PersistentFlags().StringVarP(&bandwidthLimitMode, "bandwidth_limit_mode", "", config.BandwidthLimitModeClient, "bandwidth limit mode")
 
 	rootCmd.AddCommand(xtcpCmd)
 }
@@ -70,6 +72,12 @@ var xtcpCmd = &cobra.Command{
 			cfg.Sk = sk
 			cfg.LocalIP = localIP
 			cfg.LocalPort = localPort
+			cfg.BandwidthLimit, err = config.NewBandwidthQuantity(bandwidthLimit)
+			if err != nil {
+				fmt.Println(err)
+				os.Exit(1)
+			}
+			cfg.BandwidthLimitMode = bandwidthLimitMode
 			err = cfg.CheckForCli()
 			if err != nil {
 				fmt.Println(err)

cmd/frps/main.go

@@ -26,6 +26,7 @@ import (
 
 func main() {
 	crypto.DefaultSalt = "frp"
+	// TODO: remove this when we drop support for go1.19
 	rand.Seed(time.Now().UnixNano())
 
 	Execute()

cmd/frps/root.go

@@ -39,7 +39,6 @@ var (
 
 	bindAddr             string
 	bindPort             int
-	bindUDPPort          int
 	kcpBindPort          int
 	proxyBindAddr        string
 	vhostHTTPPort        int
@@ -70,13 +69,12 @@ func init() {
 
 	rootCmd.PersistentFlags().StringVarP(&bindAddr, "bind_addr", "", "0.0.0.0", "bind address")
 	rootCmd.PersistentFlags().IntVarP(&bindPort, "bind_port", "p", 7000, "bind port")
-	rootCmd.PersistentFlags().IntVarP(&bindUDPPort, "bind_udp_port", "", 0, "bind udp port")
 	rootCmd.PersistentFlags().IntVarP(&kcpBindPort, "kcp_bind_port", "", 0, "kcp bind udp port")
 	rootCmd.PersistentFlags().StringVarP(&proxyBindAddr, "proxy_bind_addr", "", "0.0.0.0", "proxy bind address")
 	rootCmd.PersistentFlags().IntVarP(&vhostHTTPPort, "vhost_http_port", "", 0, "vhost http port")
 	rootCmd.PersistentFlags().IntVarP(&vhostHTTPSPort, "vhost_https_port", "", 0, "vhost https port")
 	rootCmd.PersistentFlags().Int64VarP(&vhostHTTPTimeout, "vhost_http_timeout", "", 60, "vhost http response header timeout")
-	rootCmd.PersistentFlags().StringVarP(&dashboardAddr, "dashboard_addr", "", "0.0.0.0", "dasboard address")
+	rootCmd.PersistentFlags().StringVarP(&dashboardAddr, "dashboard_addr", "", "0.0.0.0", "dashboard address")
 	rootCmd.PersistentFlags().IntVarP(&dashboardPort, "dashboard_port", "", 0, "dashboard port")
 	rootCmd.PersistentFlags().StringVarP(&dashboardUser, "dashboard_user", "", "admin", "dashboard user")
 	rootCmd.PersistentFlags().StringVarP(&dashboardPwd, "dashboard_pwd", "", "admin", "dashboard password")
@@ -159,7 +157,6 @@ func parseServerCommonCfgFromCmd() (cfg config.ServerCommonConf, err error) {
 
 	cfg.BindAddr = bindAddr
 	cfg.BindPort = bindPort
-	cfg.BindUDPPort = bindUDPPort
 	cfg.KCPBindPort = kcpBindPort
 	cfg.ProxyBindAddr = proxyBindAddr
 	cfg.VhostHTTPPort = vhostHTTPPort

conf/frpc_full.ini

@@ -6,6 +6,9 @@
 server_addr = 0.0.0.0
 server_port = 7000
 
+# STUN server to help penetrate NAT hole.
+# nat_hole_stun_server = stun.easyvoip.com:3478
+
 # The maximum amount of time a dial to server will wait for a connect to complete. Default value is 10 seconds.
 # dial_server_timeout = 10
 
@@ -154,6 +157,8 @@ local_ip = 127.0.0.1
 local_port = 22
 # limit bandwidth for this proxy, unit is KB and MB
 bandwidth_limit = 1MB
+# where to limit bandwidth, can be 'client' or 'server', default is 'client'
+bandwidth_limit_mode = client
 # true or false, if true, messages between frps and frpc will be encrypted, default is false
 use_encryption = false
 # if true, message will be compressed
@@ -245,7 +250,7 @@ local_ip = 127.0.0.1
 local_port = 8000
 use_encryption = false
 use_compression = false
-subdomain = web01
+subdomain = web02
 custom_domains = web02.yourdomain.com
 # if not empty, frpc will use proxy protocol to transfer connection info to your local service
 # v1 or v2 or empty
@@ -353,6 +358,11 @@ bind_addr = 127.0.0.1
 bind_port = 9001
 use_encryption = false
 use_compression = false
+# when automatic tunnel persistence is required, set it to true
+keep_tunnel_open = false
+# effective when keep_tunnel_open is set to true, the number of attempts to punch through per hour
+max_retries_an_hour = 8
+min_retry_interval = 90
 
 [tcpmuxhttpconnect]
 type = tcpmux

conf/frps_full.ini

@@ -6,9 +6,6 @@
 bind_addr = 0.0.0.0
 bind_port = 7000
 
-# udp port to help make udp hole to penetrate nat
-bind_udp_port = 7001
-
 # udp port used for kcp protocol, it can be same with 'bind_port'.
 # if not set, kcp is disabled in frps.
 kcp_bind_port = 7000
@@ -157,6 +154,9 @@ udp_packet_size = 1500
 # Dashboard port must be set first
 pprof_enable = false
 
+# Retention time for NAT hole punching strategy data.
+nat_hole_analysis_data_reserve_hours = 168
+
 [plugin.user-manager]
 addr = 127.0.0.1:9000
 path = /handler

doc/server_plugin.md

@@ -110,6 +110,8 @@ Create new proxy
         "proxy_type": <string>,
         "use_encryption": <bool>,
         "use_compression": <bool>,
+        "bandwidth_limit": <string>,
+        "bandwidth_limit_mode": <string>,
         "group": <string>,
         "group_key": <string>,
 

dockerfiles/Dockerfile-for-frpc

@@ -1,4 +1,4 @@
-FROM golang:1.19 AS building
+FROM golang:1.20 AS building
 
 COPY . /building
 WORKDIR /building

dockerfiles/Dockerfile-for-frps

@@ -1,4 +1,4 @@
-FROM golang:1.19 AS building
+FROM golang:1.20 AS building
 
 COPY . /building
 WORKDIR /building

go.mod

@@ -1,32 +1,35 @@
 module github.com/fatedier/frp
 
-go 1.19
+go 1.20
 
 require (
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
 	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/fatedier/beego v0.0.0-20171024143340-6c6a4f5bd5eb
-	github.com/fatedier/golib v0.1.1-0.20220321042308-c306138b83ac
+	github.com/fatedier/golib v0.1.1-0.20230320133937-a7edcc8c793d
 	github.com/fatedier/kcp-go v2.0.4-0.20190803094908-fe8645b0a904+incompatible
 	github.com/go-playground/validator/v10 v10.11.0
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.5.0
 	github.com/hashicorp/yamux v0.1.1
-	github.com/lucas-clemente/quic-go v0.31.0
-	github.com/onsi/ginkgo v1.16.4
-	github.com/onsi/gomega v1.20.2
+	github.com/onsi/ginkgo/v2 v2.8.3
+	github.com/onsi/gomega v1.27.0
+	github.com/pion/stun v0.4.0
 	github.com/pires/go-proxyproto v0.6.2
 	github.com/prometheus/client_golang v1.13.0
+	github.com/quic-go/quic-go v0.34.0
 	github.com/rodaine/table v1.0.1
+	github.com/samber/lo v1.38.1
 	github.com/spf13/cobra v1.1.3
-	github.com/stretchr/testify v1.7.0
-	golang.org/x/net v0.4.0
+	github.com/stretchr/testify v1.8.1
+	golang.org/x/net v0.7.0
 	golang.org/x/oauth2 v0.3.0
+	golang.org/x/sync v0.1.0
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8
 	gopkg.in/ini.v1 v1.67.0
-	k8s.io/apimachinery v0.25.0
-	k8s.io/client-go v0.25.0
+	k8s.io/apimachinery v0.26.1
+	k8s.io/client-go v0.26.1
 )
 
 require (
@@ -34,43 +37,41 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-playground/locales v0.14.0 // indirect
 	github.com/go-playground/universal-translator v0.18.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
-	github.com/google/go-cmp v0.5.8 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.6 // indirect
 	github.com/klauspost/reedsolomon v1.9.15 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect
-	github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo/v2 v2.2.0 // indirect
+	github.com/pion/transport/v2 v2.0.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/quic-go/qtls-go1-19 v0.3.2 // indirect
+	github.com/quic-go/qtls-go1-20 v0.2.2 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 // indirect
 	github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b // indirect
 	github.com/tjfoc/gmsm v1.4.1 // indirect
 	golang.org/x/crypto v0.4.0 // indirect
-	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
-	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/text v0.5.0 // indirect
-	golang.org/x/tools v0.1.12 // indirect
+	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
+	golang.org/x/mod v0.8.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/tools v0.6.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
+	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
 )

go.sum

@@ -121,14 +121,12 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatedier/beego v0.0.0-20171024143340-6c6a4f5bd5eb h1:wCrNShQidLmvVWn/0PikGmpdP0vtQmnvyRg3ZBEhczw=
 github.com/fatedier/beego v0.0.0-20171024143340-6c6a4f5bd5eb/go.mod h1:wx3gB6dbIfBRcucp94PI9Bt3I0F2c/MyNEWuhzpWiwk=
-github.com/fatedier/golib v0.1.1-0.20220321042308-c306138b83ac h1:td1FJwN/oz8+9GldeEm3YdBX0Husc0FSPywLesZxi4w=
-github.com/fatedier/golib v0.1.1-0.20220321042308-c306138b83ac/go.mod h1:fLV0TLwHqrnB/L3jbNl67Gn6PCLggDGHniX1wLrA2Qo=
+github.com/fatedier/golib v0.1.1-0.20230320133937-a7edcc8c793d h1:/m9Atycn9uKRwwOkxv4c+zaugxRgkdSG/Eg3IJWOpNs=
+github.com/fatedier/golib v0.1.1-0.20230320133937-a7edcc8c793d/go.mod h1:Wdn1pJ0dHB1lah6FPYwt4AO9NEmWI0OzW13dpzC9g4E=
 github.com/fatedier/kcp-go v2.0.4-0.20190803094908-fe8645b0a904+incompatible h1:ssXat9YXFvigNge/IkkZvFMn8yeYKFX+uI6wn2mLJ74=
 github.com/fatedier/kcp-go v2.0.4-0.20190803094908-fe8645b0a904+incompatible/go.mod h1:YpCOaxj7vvMThhIQ9AfTOPW2sfztQR5WDfs7AflSy4s=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
-github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -141,7 +139,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
-github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -205,8 +204,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -272,7 +272,6 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE=
 github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -307,13 +306,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
-github.com/lucas-clemente/quic-go v0.31.0 h1:MfNp3fk0wjWRajw6quMFA3ap1AVtlU+2mtwmbVogB2M=
-github.com/lucas-clemente/quic-go v0.31.0/go.mod h1:0wFbizLgYzqHqtlyxyCaJKlE7bYgE6JQ+54TLd/Dq2g=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/marten-seemann/qtls-go1-18 v0.1.3 h1:R4H2Ks8P6pAtUagjFty2p7BVHn3XiwDAl7TTQf5h7TI=
-github.com/marten-seemann/qtls-go1-18 v0.1.3/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
-github.com/marten-seemann/qtls-go1-19 v0.1.1 h1:mnbxeq3oEyQxQXwI4ReCgW9DPoPR94sNlqWoDZnjRIE=
-github.com/marten-seemann/qtls-go1-19 v0.1.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
@@ -336,22 +329,18 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
-github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
-github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
-github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI=
-github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
-github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY=
-github.com/onsi/gomega v1.20.2/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
+github.com/onsi/ginkgo/v2 v2.8.3 h1:RpbK1G8nWPNaCVFBWsOGnEQQGgASi6b8fxcWBvDYjxQ=
+github.com/onsi/ginkgo/v2 v2.8.3/go.mod h1:6OaUA8BCi0aZfmzYT/q9AacwTzDpNbxILUT+TlBq6MY=
+github.com/onsi/gomega v1.27.0 h1:QLidEla4bXUuZVFa4KX6JHCsuGgbi85LC/pCHrt/O08=
+github.com/onsi/gomega v1.27.0/go.mod h1:i189pavgK95OSIipFBa74gC2V4qrQuvjuyGEr3GmbXA=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
+github.com/pion/stun v0.4.0 h1:vgRrbBE2htWHy7l3Zsxckk7rkjnjOsSM7PHZnBwo8rk=
+github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw=
+github.com/pion/transport/v2 v2.0.0 h1:bsMYyqHCbkvHwj+eNCFBuxtlKndKfyGI2vaQmM3fIE4=
+github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc=
 github.com/pires/go-proxyproto v0.6.2 h1:KAZ7UteSOt6urjme6ZldyFm4wDe/z0ZUP0Yv0Dos0d8=
 github.com/pires/go-proxyproto v0.6.2/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
@@ -392,6 +381,12 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U=
+github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI=
+github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E=
+github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM=
+github.com/quic-go/quic-go v0.34.0 h1:OvOJ9LFjTySgwOTYUZmNoq0FzVicP8YujpV0kB7m2lU=
+github.com/quic-go/quic-go v0.34.0/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g=
 github.com/rodaine/table v1.0.1 h1:U/VwCnUxlVYxw8+NJiLIuCxA/xa6jL38MY3FYysVWWQ=
 github.com/rodaine/table v1.0.1/go.mod h1:UVEtfBsflpeEcD56nF4F5AocNFta0ZuolpSVdPtlmP4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -402,6 +397,8 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
+github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -412,7 +409,6 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
@@ -423,13 +419,18 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 h1:89CEmDvlq/F7SJEOqkIdNDGJXrQIhuIx9D2DBXjavSU=
 github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU=
@@ -445,6 +446,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -465,6 +467,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
 golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
@@ -478,8 +481,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
-golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
+golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o=
+golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -505,11 +508,11 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -535,7 +538,6 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -559,9 +561,11 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
-golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -597,10 +601,12 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -613,10 +619,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -640,7 +643,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -672,11 +674,15 @@ golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -686,8 +692,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -740,7 +747,6 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u
 golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
@@ -748,8 +754,9 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -937,15 +944,12 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
 gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
@@ -965,13 +969,12 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU=
-k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0=
-k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E=
-k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4=
-k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ=
+k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
+k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU=
+k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE=
+k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs=
+k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

hack/run-e2e.sh

@@ -5,7 +5,7 @@ ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
 which ginkgo &> /dev/null
 if [ $? -ne 0 ]; then
     echo "ginkgo not found, try to install..."
-    go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+    go install github.com/onsi/ginkgo/v2/ginkgo@v2.8.3
 fi
 
 debug=false
@@ -17,4 +17,4 @@ if [ x${LOG_LEVEL} != x"" ]; then
     logLevel=${LOG_LEVEL}
 fi
 
-ginkgo -nodes=8 -slowSpecThreshold=20 ${ROOT}/test/e2e -- -frpc-path=${ROOT}/bin/frpc -frps-path=${ROOT}/bin/frps -log-level=${logLevel} -debug=${debug}
+ginkgo -nodes=8 --poll-progress-after=30s ${ROOT}/test/e2e -- -frpc-path=${ROOT}/bin/frpc -frps-path=${ROOT}/bin/frps -log-level=${logLevel} -debug=${debug}

pkg/auth/token.go

@@ -73,30 +73,30 @@ func (auth *TokenAuthSetterVerifier) SetNewWorkConn(newWorkConnMsg *msg.NewWorkC
 	return nil
 }
 
-func (auth *TokenAuthSetterVerifier) VerifyLogin(loginMsg *msg.Login) error {
-	if util.GetAuthKey(auth.token, loginMsg.Timestamp) != loginMsg.PrivilegeKey {
+func (auth *TokenAuthSetterVerifier) VerifyLogin(m *msg.Login) error {
+	if !util.ConstantTimeEqString(util.GetAuthKey(auth.token, m.Timestamp), m.PrivilegeKey) {
 		return fmt.Errorf("token in login doesn't match token from configuration")
 	}
 	return nil
 }
 
-func (auth *TokenAuthSetterVerifier) VerifyPing(pingMsg *msg.Ping) error {
+func (auth *TokenAuthSetterVerifier) VerifyPing(m *msg.Ping) error {
 	if !auth.AuthenticateHeartBeats {
 		return nil
 	}
 
-	if util.GetAuthKey(auth.token, pingMsg.Timestamp) != pingMsg.PrivilegeKey {
+	if !util.ConstantTimeEqString(util.GetAuthKey(auth.token, m.Timestamp), m.PrivilegeKey) {
 		return fmt.Errorf("token in heartbeat doesn't match token from configuration")
 	}
 	return nil
 }
 
-func (auth *TokenAuthSetterVerifier) VerifyNewWorkConn(newWorkConnMsg *msg.NewWorkConn) error {
+func (auth *TokenAuthSetterVerifier) VerifyNewWorkConn(m *msg.NewWorkConn) error {
 	if !auth.AuthenticateNewWorkConns {
 		return nil
 	}
 
-	if util.GetAuthKey(auth.token, newWorkConnMsg.Timestamp) != newWorkConnMsg.PrivilegeKey {
+	if !util.ConstantTimeEqString(util.GetAuthKey(auth.token, m.Timestamp), m.PrivilegeKey) {
 		return fmt.Errorf("token in NewWorkConn doesn't match token from configuration")
 	}
 	return nil

pkg/config/client.go

@@ -38,6 +38,8 @@ type ClientCommonConf struct {
 	// ServerPort specifies the port to connect to the server on. By default,
 	// this value is 7000.
 	ServerPort int `ini:"server_port" json:"server_port"`
+	// STUN server to help penetrate NAT hole.
+	NatHoleSTUNServer string `ini:"nat_hole_stun_server" json:"nat_hole_stun_server"`
 	// The maximum amount of time a dial to server will wait for a connect to complete.
 	DialServerTimeout int64 `ini:"dial_server_timeout" json:"dial_server_timeout"`
 	// DialServerKeepAlive specifies the interval between keep-alive probes for an active network connection between frpc and frps.
@@ -169,6 +171,7 @@ func GetDefaultClientConf() ClientCommonConf {
 		ClientConfig:            auth.GetDefaultClientConf(),
 		ServerAddr:              "0.0.0.0",
 		ServerPort:              7000,
+		NatHoleSTUNServer:       "stun.easyvoip.com:3478",
 		DialServerTimeout:       10,
 		DialServerKeepAlive:     7200,
 		HTTPProxy:               os.Getenv("http_proxy"),

pkg/config/client_test.go

@@ -74,6 +74,7 @@ var testClientBytesWithFull = []byte(`
 		local_ip = 127.0.0.9
 		local_port = 29
 		bandwidth_limit = 19MB
+		bandwidth_limit_mode = server
 		use_encryption
 		use_compression
 		remote_port = 6009
@@ -259,6 +260,7 @@ func Test_LoadClientCommonConf(t *testing.T) {
 		},
 		ServerAddr:              "0.0.0.9",
 		ServerPort:              7009,
+		NatHoleSTUNServer:       "stun.easyvoip.com:3478",
 		DialServerTimeout:       10,
 		DialServerKeepAlive:     7200,
 		HTTPProxy:               "http://user:passwd@192.168.1.128:8080",
@@ -309,13 +311,14 @@ func Test_LoadClientBasicConf(t *testing.T) {
 	proxyExpected := map[string]ProxyConf{
 		testUser + ".ssh": &TCPProxyConf{
 			BaseProxyConf: BaseProxyConf{
-				ProxyName:      testUser + ".ssh",
-				ProxyType:      consts.TCPProxy,
-				UseCompression: true,
-				UseEncryption:  true,
-				Group:          "test_group",
-				GroupKey:       "123456",
-				BandwidthLimit: MustBandwidthQuantity("19MB"),
+				ProxyName:          testUser + ".ssh",
+				ProxyType:          consts.TCPProxy,
+				UseCompression:     true,
+				UseEncryption:      true,
+				Group:              "test_group",
+				GroupKey:           "123456",
+				BandwidthLimit:     MustBandwidthQuantity("19MB"),
+				BandwidthLimitMode: BandwidthLimitModeServer,
 				Metas: map[string]string{
 					"var1": "123",
 					"var2": "234",
@@ -342,6 +345,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "127.0.0.9",
 					LocalPort: 29,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 9,
 		},
@@ -353,6 +357,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "127.0.0.9",
 					LocalPort: 6010,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6010,
 		},
@@ -364,6 +369,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "127.0.0.9",
 					LocalPort: 6011,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6011,
 		},
@@ -375,6 +381,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "127.0.0.9",
 					LocalPort: 6019,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6019,
 		},
@@ -388,6 +395,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "114.114.114.114",
 					LocalPort: 59,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6009,
 		},
@@ -401,6 +409,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "114.114.114.114",
 					LocalPort: 6000,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6000,
 		},
@@ -414,6 +423,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "114.114.114.114",
 					LocalPort: 6010,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6010,
 		},
@@ -427,6 +437,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "114.114.114.114",
 					LocalPort: 6011,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6011,
 		},
@@ -447,6 +458,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					HealthCheckIntervalS: 19,
 					HealthCheckURL:       "http://127.0.0.9:89/status",
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			DomainConf: DomainConf{
 				CustomDomains: []string{"web02.yourdomain.com"},
@@ -471,6 +483,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalPort: 8009,
 				},
 				ProxyProtocolVersion: "v2",
+				BandwidthLimitMode:   BandwidthLimitModeClient,
 			},
 			DomainConf: DomainConf{
 				CustomDomains: []string{"web02.yourdomain.com"},
@@ -485,6 +498,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "127.0.0.1",
 					LocalPort: 22,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			Role: "server",
 			Sk:   "abcdefg",
@@ -497,6 +511,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "127.0.0.1",
 					LocalPort: 22,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			Role: "server",
 			Sk:   "abcdefg",
@@ -509,6 +524,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 					LocalIP:   "127.0.0.1",
 					LocalPort: 10701,
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			DomainConf: DomainConf{
 				CustomDomains: []string{"tunnel1"},
@@ -527,6 +543,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 						"plugin_unix_path": "/var/run/docker.sock",
 					},
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6003,
 		},
@@ -542,6 +559,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 						"plugin_http_passwd": "abc",
 					},
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6004,
 		},
@@ -557,6 +575,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 						"plugin_passwd": "abc",
 					},
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6005,
 		},
@@ -574,6 +593,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 						"plugin_http_passwd":  "abc",
 					},
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			RemotePort: 6006,
 		},
@@ -592,6 +612,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 						"plugin_header_X-From-Where": "frp",
 					},
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			DomainConf: DomainConf{
 				CustomDomains: []string{"test.yourdomain.com"},
@@ -610,6 +631,7 @@ func Test_LoadClientBasicConf(t *testing.T) {
 						"plugin_header_X-From-Where": "frp",
 					},
 				},
+				BandwidthLimitMode: BandwidthLimitModeClient,
 			},
 			DomainConf: DomainConf{
 				CustomDomains: []string{"test.yourdomain.com"},
@@ -639,6 +661,9 @@ func Test_LoadClientBasicConf(t *testing.T) {
 				BindAddr:   "127.0.0.1",
 				BindPort:   9001,
 			},
+			Protocol:         "quic",
+			MaxRetriesAnHour: 8,
+			MinRetryInterval: 90,
 		},
 	}
 

pkg/config/proxy.go

@@ -141,6 +141,10 @@ type BaseProxyConf struct {
 	// BandwidthLimit limit the bandwidth
 	// 0 means no limit
 	BandwidthLimit BandwidthQuantity `ini:"bandwidth_limit" json:"bandwidth_limit"`
+	// BandwidthLimitMode specifies whether to limit the bandwidth on the
+	// client or server side. Valid values include "client" and "server".
+	// By default, this value is "client".
+	BandwidthLimitMode string `ini:"bandwidth_limit_mode" json:"bandwidth_limit_mode"`
 
 	// meta info for each proxy
 	Metas map[string]string `ini:"-" json:"metas"`
@@ -183,6 +187,8 @@ type TCPProxyConf struct {
 type TCPMuxProxyConf struct {
 	BaseProxyConf   `ini:",extends"`
 	DomainConf      `ini:",extends"`
+	HTTPUser        string `ini:"http_user" json:"http_user,omitempty"`
+	HTTPPwd         string `ini:"http_pwd" json:"http_pwd,omitempty"`
 	RouteByHTTPUser string `ini:"route_by_http_user" json:"route_by_http_user"`
 
 	Multiplexer string `ini:"multiplexer"`
@@ -319,6 +325,7 @@ func defaultBaseProxyConf(proxyType string) BaseProxyConf {
 		LocalSvrConf: LocalSvrConf{
 			LocalIP: "127.0.0.1",
 		},
+		BandwidthLimitMode: BandwidthLimitModeClient,
 	}
 }
 
@@ -335,6 +342,7 @@ func (cfg *BaseProxyConf) compare(cmp *BaseProxyConf) bool {
 		cfg.GroupKey != cmp.GroupKey ||
 		cfg.ProxyProtocolVersion != cmp.ProxyProtocolVersion ||
 		!cfg.BandwidthLimit.Equal(&cmp.BandwidthLimit) ||
+		cfg.BandwidthLimitMode != cmp.BandwidthLimitMode ||
 		!reflect.DeepEqual(cfg.Metas, cmp.Metas) {
 		return false
 	}
@@ -389,6 +397,11 @@ func (cfg *BaseProxyConf) marshalToMsg(pMsg *msg.NewProxy) {
 	pMsg.ProxyType = cfg.ProxyType
 	pMsg.UseEncryption = cfg.UseEncryption
 	pMsg.UseCompression = cfg.UseCompression
+	pMsg.BandwidthLimit = cfg.BandwidthLimit.String()
+	// leave it empty for default value to reduce traffic
+	if cfg.BandwidthLimitMode != "client" {
+		pMsg.BandwidthLimitMode = cfg.BandwidthLimitMode
+	}
 	pMsg.Group = cfg.Group
 	pMsg.GroupKey = cfg.GroupKey
 	pMsg.Metas = cfg.Metas
@@ -399,6 +412,12 @@ func (cfg *BaseProxyConf) unmarshalFromMsg(pMsg *msg.NewProxy) {
 	cfg.ProxyType = pMsg.ProxyType
 	cfg.UseEncryption = pMsg.UseEncryption
 	cfg.UseCompression = pMsg.UseCompression
+	if pMsg.BandwidthLimit != "" {
+		cfg.BandwidthLimit, _ = NewBandwidthQuantity(pMsg.BandwidthLimit)
+	}
+	if pMsg.BandwidthLimitMode != "" {
+		cfg.BandwidthLimitMode = pMsg.BandwidthLimitMode
+	}
 	cfg.Group = pMsg.Group
 	cfg.GroupKey = pMsg.GroupKey
 	cfg.Metas = pMsg.Metas
@@ -411,6 +430,10 @@ func (cfg *BaseProxyConf) checkForCli() (err error) {
 		}
 	}
 
+	if cfg.BandwidthLimitMode != "client" && cfg.BandwidthLimitMode != "server" {
+		return fmt.Errorf("bandwidth_limit_mode should be client or server")
+	}
+
 	if err = cfg.LocalSvrConf.checkForCli(); err != nil {
 		return
 	}
@@ -420,6 +443,13 @@ func (cfg *BaseProxyConf) checkForCli() (err error) {
 	return nil
 }
 
+func (cfg *BaseProxyConf) checkForSvr() (err error) {
+	if cfg.BandwidthLimitMode != "client" && cfg.BandwidthLimitMode != "server" {
+		return fmt.Errorf("bandwidth_limit_mode should be client or server")
+	}
+	return nil
+}
+
 // DomainConf
 func (cfg *DomainConf) check() (err error) {
 	if len(cfg.CustomDomains) == 0 && cfg.SubDomain == "" {
@@ -557,6 +587,9 @@ func (cfg *TCPProxyConf) CheckForCli() (err error) {
 }
 
 func (cfg *TCPProxyConf) CheckForSvr(serverCfg ServerCommonConf) error {
+	if err := cfg.BaseProxyConf.checkForSvr(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -576,7 +609,10 @@ func (cfg *TCPMuxProxyConf) Compare(cmp ProxyConf) bool {
 		return false
 	}
 
-	if cfg.Multiplexer != cmpConf.Multiplexer || cfg.RouteByHTTPUser != cmpConf.RouteByHTTPUser {
+	if cfg.Multiplexer != cmpConf.Multiplexer ||
+		cfg.HTTPUser != cmpConf.HTTPUser ||
+		cfg.HTTPPwd != cmpConf.HTTPPwd ||
+		cfg.RouteByHTTPUser != cmpConf.RouteByHTTPUser {
 		return false
 	}
 
@@ -601,6 +637,8 @@ func (cfg *TCPMuxProxyConf) UnmarshalFromMsg(pMsg *msg.NewProxy) {
 	cfg.CustomDomains = pMsg.CustomDomains
 	cfg.SubDomain = pMsg.SubDomain
 	cfg.Multiplexer = pMsg.Multiplexer
+	cfg.HTTPUser = pMsg.HTTPUser
+	cfg.HTTPPwd = pMsg.HTTPPwd
 	cfg.RouteByHTTPUser = pMsg.RouteByHTTPUser
 }
 
@@ -611,6 +649,8 @@ func (cfg *TCPMuxProxyConf) MarshalToMsg(pMsg *msg.NewProxy) {
 	pMsg.CustomDomains = cfg.CustomDomains
 	pMsg.SubDomain = cfg.SubDomain
 	pMsg.Multiplexer = cfg.Multiplexer
+	pMsg.HTTPUser = cfg.HTTPUser
+	pMsg.HTTPPwd = cfg.HTTPPwd
 	pMsg.RouteByHTTPUser = cfg.RouteByHTTPUser
 }
 
@@ -632,6 +672,10 @@ func (cfg *TCPMuxProxyConf) CheckForCli() (err error) {
 }
 
 func (cfg *TCPMuxProxyConf) CheckForSvr(serverCfg ServerCommonConf) (err error) {
+	if err := cfg.BaseProxyConf.checkForSvr(); err != nil {
+		return err
+	}
+
 	if cfg.Multiplexer != consts.HTTPConnectTCPMultiplexer {
 		return fmt.Errorf("proxy [%s] incorrect multiplexer [%s]", cfg.ProxyName, cfg.Multiplexer)
 	}
@@ -703,6 +747,9 @@ func (cfg *UDPProxyConf) CheckForCli() (err error) {
 }
 
 func (cfg *UDPProxyConf) CheckForSvr(serverCfg ServerCommonConf) error {
+	if err := cfg.BaseProxyConf.checkForSvr(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -788,6 +835,10 @@ func (cfg *HTTPProxyConf) CheckForCli() (err error) {
 }
 
 func (cfg *HTTPProxyConf) CheckForSvr(serverCfg ServerCommonConf) (err error) {
+	if err := cfg.BaseProxyConf.checkForSvr(); err != nil {
+		return err
+	}
+
 	if serverCfg.VhostHTTPPort == 0 {
 		return fmt.Errorf("type [http] not support when vhost_http_port is not set")
 	}
@@ -860,6 +911,10 @@ func (cfg *HTTPSProxyConf) CheckForCli() (err error) {
 }
 
 func (cfg *HTTPSProxyConf) CheckForSvr(serverCfg ServerCommonConf) (err error) {
+	if err := cfg.BaseProxyConf.checkForSvr(); err != nil {
+		return err
+	}
+
 	if serverCfg.VhostHTTPSPort == 0 {
 		return fmt.Errorf("type [https] not support when vhost_https_port is not set")
 	}
@@ -932,6 +987,9 @@ func (cfg *SUDPProxyConf) CheckForCli() (err error) {
 }
 
 func (cfg *SUDPProxyConf) CheckForSvr(serverCfg ServerCommonConf) error {
+	if err := cfg.BaseProxyConf.checkForSvr(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -998,6 +1056,9 @@ func (cfg *STCPProxyConf) CheckForCli() (err error) {
 }
 
 func (cfg *STCPProxyConf) CheckForSvr(serverCfg ServerCommonConf) error {
+	if err := cfg.BaseProxyConf.checkForSvr(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -1017,7 +1078,6 @@ func (cfg *XTCPProxyConf) Compare(cmp ProxyConf) bool {
 		cfg.Sk != cmpConf.Sk {
 		return false
 	}
-
 	return true
 }
 
@@ -1031,7 +1091,6 @@ func (cfg *XTCPProxyConf) UnmarshalFromIni(prefix string, name string, section *
 	if cfg.Role == "" {
 		cfg.Role = "server"
 	}
-
 	return nil
 }
 
@@ -1059,10 +1118,12 @@ func (cfg *XTCPProxyConf) CheckForCli() (err error) {
 	if cfg.Role != "server" {
 		return fmt.Errorf("role should be 'server'")
 	}
-
 	return
 }
 
 func (cfg *XTCPProxyConf) CheckForSvr(serverCfg ServerCommonConf) error {
+	if err := cfg.BaseProxyConf.checkForSvr(); err != nil {
+		return err
+	}
 	return nil
 }

pkg/config/proxy_test.go

@@ -58,6 +58,7 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 				local_ip = 127.0.0.9
 				local_port = 29
 				bandwidth_limit = 19MB
+				bandwidth_limit_mode = server
 				use_encryption
 				use_compression
 				remote_port = 6009
@@ -71,13 +72,14 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 				meta_var2 = 234`),
 			expected: &TCPProxyConf{
 				BaseProxyConf: BaseProxyConf{
-					ProxyName:      testProxyPrefix + "ssh",
-					ProxyType:      consts.TCPProxy,
-					UseCompression: true,
-					UseEncryption:  true,
-					Group:          "test_group",
-					GroupKey:       "123456",
-					BandwidthLimit: MustBandwidthQuantity("19MB"),
+					ProxyName:          testProxyPrefix + "ssh",
+					ProxyType:          consts.TCPProxy,
+					UseCompression:     true,
+					UseEncryption:      true,
+					Group:              "test_group",
+					GroupKey:           "123456",
+					BandwidthLimit:     MustBandwidthQuantity("19MB"),
+					BandwidthLimitMode: BandwidthLimitModeServer,
 					Metas: map[string]string{
 						"var1": "123",
 						"var2": "234",
@@ -114,6 +116,7 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 						LocalIP:   "127.0.0.9",
 						LocalPort: 29,
 					},
+					BandwidthLimitMode: BandwidthLimitModeClient,
 				},
 				RemotePort: 9,
 			},
@@ -139,6 +142,7 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 						LocalIP:   "114.114.114.114",
 						LocalPort: 59,
 					},
+					BandwidthLimitMode: BandwidthLimitModeClient,
 				},
 				RemotePort: 6009,
 			},
@@ -182,6 +186,7 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 						HealthCheckIntervalS: 19,
 						HealthCheckURL:       "http://127.0.0.9:89/status",
 					},
+					BandwidthLimitMode: BandwidthLimitModeClient,
 				},
 				DomainConf: DomainConf{
 					CustomDomains: []string{"web02.yourdomain.com"},
@@ -220,6 +225,7 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 						LocalPort: 8009,
 					},
 					ProxyProtocolVersion: "v2",
+					BandwidthLimitMode:   BandwidthLimitModeClient,
 				},
 				DomainConf: DomainConf{
 					CustomDomains: []string{"web02.yourdomain.com"},
@@ -246,6 +252,7 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 						LocalIP:   "127.0.0.1",
 						LocalPort: 22,
 					},
+					BandwidthLimitMode: BandwidthLimitModeClient,
 				},
 				Role: "server",
 				Sk:   "abcdefg",
@@ -270,6 +277,7 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 						LocalIP:   "127.0.0.1",
 						LocalPort: 22,
 					},
+					BandwidthLimitMode: BandwidthLimitModeClient,
 				},
 				Role: "server",
 				Sk:   "abcdefg",
@@ -293,6 +301,7 @@ func Test_Proxy_UnmarshalFromIni(t *testing.T) {
 						LocalIP:   "127.0.0.1",
 						LocalPort: 10701,
 					},
+					BandwidthLimitMode: BandwidthLimitModeClient,
 				},
 				DomainConf: DomainConf{
 					CustomDomains: []string{"tunnel1"},
@@ -347,6 +356,7 @@ func Test_RangeProxy_UnmarshalFromIni(t *testing.T) {
 							LocalIP:   "127.0.0.9",
 							LocalPort: 6010,
 						},
+						BandwidthLimitMode: BandwidthLimitModeClient,
 					},
 					RemotePort: 6010,
 				},
@@ -358,6 +368,7 @@ func Test_RangeProxy_UnmarshalFromIni(t *testing.T) {
 							LocalIP:   "127.0.0.9",
 							LocalPort: 6011,
 						},
+						BandwidthLimitMode: BandwidthLimitModeClient,
 					},
 					RemotePort: 6011,
 				},
@@ -369,6 +380,7 @@ func Test_RangeProxy_UnmarshalFromIni(t *testing.T) {
 							LocalIP:   "127.0.0.9",
 							LocalPort: 6019,
 						},
+						BandwidthLimitMode: BandwidthLimitModeClient,
 					},
 					RemotePort: 6019,
 				},
@@ -396,6 +408,7 @@ func Test_RangeProxy_UnmarshalFromIni(t *testing.T) {
 							LocalIP:   "114.114.114.114",
 							LocalPort: 6000,
 						},
+						BandwidthLimitMode: BandwidthLimitModeClient,
 					},
 					RemotePort: 6000,
 				},
@@ -409,6 +422,7 @@ func Test_RangeProxy_UnmarshalFromIni(t *testing.T) {
 							LocalIP:   "114.114.114.114",
 							LocalPort: 6010,
 						},
+						BandwidthLimitMode: BandwidthLimitModeClient,
 					},
 					RemotePort: 6010,
 				},
@@ -422,6 +436,7 @@ func Test_RangeProxy_UnmarshalFromIni(t *testing.T) {
 							LocalIP:   "114.114.114.114",
 							LocalPort: 6011,
 						},
+						BandwidthLimitMode: BandwidthLimitModeClient,
 					},
 					RemotePort: 6011,
 				},

pkg/config/server.go

@@ -38,10 +38,6 @@ type ServerCommonConf struct {
 	// BindPort specifies the port that the server listens on. By default, this
 	// value is 7000.
 	BindPort int `ini:"bind_port" json:"bind_port" validate:"gte=0,lte=65535"`
-	// BindUDPPort specifies the UDP port that the server listens on. If this
-	// value is 0, the server will not listen for UDP connections. By default,
-	// this value is 0
-	BindUDPPort int `ini:"bind_udp_port" json:"bind_udp_port" validate:"gte=0,lte=65535"`
 	// KCPBindPort specifies the KCP port that the server listens on. If this
 	// value is 0, the server will not listen for KCP connections. By default,
 	// this value is 0.
@@ -154,6 +150,8 @@ type ServerCommonConf struct {
 	// If the length of this value is 0, all ports are allowed. By default,
 	// this value is an empty set.
 	AllowPorts map[int]struct{} `ini:"-" json:"-"`
+	// Original string.
+	AllowPortsStr string `ini:"-" json:"-"`
 	// MaxPoolCount specifies the maximum pool size for each proxy. By default,
 	// this value is 5.
 	MaxPoolCount int64 `ini:"max_pool_count" json:"max_pool_count"`
@@ -194,35 +192,38 @@ type ServerCommonConf struct {
 	// Enable golang pprof handlers in dashboard listener.
 	// Dashboard port must be set first.
 	PprofEnable bool `ini:"pprof_enable" json:"pprof_enable"`
+	// NatHoleAnalysisDataReserveHours specifies the hours to reserve nat hole analysis data.
+	NatHoleAnalysisDataReserveHours int64 `ini:"nat_hole_analysis_data_reserve_hours" json:"nat_hole_analysis_data_reserve_hours"`
 }
 
 // GetDefaultServerConf returns a server configuration with reasonable
 // defaults.
 func GetDefaultServerConf() ServerCommonConf {
 	return ServerCommonConf{
-		ServerConfig:            auth.GetDefaultServerConf(),
-		BindAddr:                "0.0.0.0",
-		BindPort:                7000,
-		QUICKeepalivePeriod:     10,
-		QUICMaxIdleTimeout:      30,
-		QUICMaxIncomingStreams:  100000,
-		VhostHTTPTimeout:        60,
-		DashboardAddr:           "0.0.0.0",
-		LogFile:                 "console",
-		LogWay:                  "console",
-		LogLevel:                "info",
-		LogMaxDays:              3,
-		DetailedErrorsToClient:  true,
-		TCPMux:                  true,
-		TCPMuxKeepaliveInterval: 60,
-		TCPKeepAlive:            7200,
-		AllowPorts:              make(map[int]struct{}),
-		MaxPoolCount:            5,
-		MaxPortsPerClient:       0,
-		HeartbeatTimeout:        90,
-		UserConnTimeout:         10,
-		HTTPPlugins:             make(map[string]plugin.HTTPPluginOptions),
-		UDPPacketSize:           1500,
+		ServerConfig:                    auth.GetDefaultServerConf(),
+		BindAddr:                        "0.0.0.0",
+		BindPort:                        7000,
+		QUICKeepalivePeriod:             10,
+		QUICMaxIdleTimeout:              30,
+		QUICMaxIncomingStreams:          100000,
+		VhostHTTPTimeout:                60,
+		DashboardAddr:                   "0.0.0.0",
+		LogFile:                         "console",
+		LogWay:                          "console",
+		LogLevel:                        "info",
+		LogMaxDays:                      3,
+		DetailedErrorsToClient:          true,
+		TCPMux:                          true,
+		TCPMuxKeepaliveInterval:         60,
+		TCPKeepAlive:                    7200,
+		AllowPorts:                      make(map[int]struct{}),
+		MaxPoolCount:                    5,
+		MaxPortsPerClient:               0,
+		HeartbeatTimeout:                90,
+		UserConnTimeout:                 10,
+		HTTPPlugins:                     make(map[string]plugin.HTTPPluginOptions),
+		UDPPacketSize:                   1500,
+		NatHoleAnalysisDataReserveHours: 7 * 24,
 	}
 }
 
@@ -259,6 +260,7 @@ func UnmarshalServerConfFromIni(source interface{}) (ServerCommonConf, error) {
 		for _, port := range allowPorts {
 			common.AllowPorts[int(port)] = struct{}{}
 		}
+		common.AllowPortsStr = allowPortStr
 	}
 
 	// plugin.xxx

pkg/config/server_test.go

@@ -104,7 +104,6 @@ func Test_LoadServerCommonConf(t *testing.T) {
 				},
 				BindAddr:               "0.0.0.9",
 				BindPort:               7009,
-				BindUDPPort:            7008,
 				KCPBindPort:            7007,
 				QUICKeepalivePeriod:    10,
 				QUICMaxIdleTimeout:     30,
@@ -134,17 +133,19 @@ func Test_LoadServerCommonConf(t *testing.T) {
 					12: {},
 					99: {},
 				},
-				MaxPoolCount:            59,
-				MaxPortsPerClient:       9,
-				TLSOnly:                 true,
-				TLSCertFile:             "server.crt",
-				TLSKeyFile:              "server.key",
-				TLSTrustedCaFile:        "ca.crt",
-				SubDomainHost:           "frps.com",
-				TCPMux:                  true,
-				TCPMuxKeepaliveInterval: 60,
-				TCPKeepAlive:            7200,
-				UDPPacketSize:           1509,
+				AllowPortsStr:                   "10-12,99",
+				MaxPoolCount:                    59,
+				MaxPortsPerClient:               9,
+				TLSOnly:                         true,
+				TLSCertFile:                     "server.crt",
+				TLSKeyFile:                      "server.key",
+				TLSTrustedCaFile:                "ca.crt",
+				SubDomainHost:                   "frps.com",
+				TCPMux:                          true,
+				TCPMuxKeepaliveInterval:         60,
+				TCPKeepAlive:                    7200,
+				UDPPacketSize:                   1509,
+				NatHoleAnalysisDataReserveHours: 7 * 24,
 
 				HTTPPlugins: map[string]plugin.HTTPPluginOptions{
 					"user-manager": {
@@ -179,32 +180,32 @@ func Test_LoadServerCommonConf(t *testing.T) {
 						AuthenticateNewWorkConns: false,
 					},
 				},
-				BindAddr:                "0.0.0.9",
-				BindPort:                7009,
-				BindUDPPort:             7008,
-				QUICKeepalivePeriod:     10,
-				QUICMaxIdleTimeout:      30,
-				QUICMaxIncomingStreams:  100000,
-				ProxyBindAddr:           "0.0.0.9",
-				VhostHTTPTimeout:        60,
-				DashboardAddr:           "0.0.0.0",
-				DashboardUser:           "",
-				DashboardPwd:            "",
-				EnablePrometheus:        false,
-				LogFile:                 "console",
-				LogWay:                  "console",
-				LogLevel:                "info",
-				LogMaxDays:              3,
-				DetailedErrorsToClient:  true,
-				TCPMux:                  true,
-				TCPMuxKeepaliveInterval: 60,
-				TCPKeepAlive:            7200,
-				AllowPorts:              make(map[int]struct{}),
-				MaxPoolCount:            5,
-				HeartbeatTimeout:        90,
-				UserConnTimeout:         10,
-				HTTPPlugins:             make(map[string]plugin.HTTPPluginOptions),
-				UDPPacketSize:           1500,
+				BindAddr:                        "0.0.0.9",
+				BindPort:                        7009,
+				QUICKeepalivePeriod:             10,
+				QUICMaxIdleTimeout:              30,
+				QUICMaxIncomingStreams:          100000,
+				ProxyBindAddr:                   "0.0.0.9",
+				VhostHTTPTimeout:                60,
+				DashboardAddr:                   "0.0.0.0",
+				DashboardUser:                   "",
+				DashboardPwd:                    "",
+				EnablePrometheus:                false,
+				LogFile:                         "console",
+				LogWay:                          "console",
+				LogLevel:                        "info",
+				LogMaxDays:                      3,
+				DetailedErrorsToClient:          true,
+				TCPMux:                          true,
+				TCPMuxKeepaliveInterval:         60,
+				TCPKeepAlive:                    7200,
+				AllowPorts:                      make(map[int]struct{}),
+				MaxPoolCount:                    5,
+				HeartbeatTimeout:                90,
+				UserConnTimeout:                 10,
+				HTTPPlugins:                     make(map[string]plugin.HTTPPluginOptions),
+				UDPPacketSize:                   1500,
+				NatHoleAnalysisDataReserveHours: 7 * 24,
 			},
 		},
 	}

pkg/config/types.go

@@ -24,6 +24,9 @@ import (
 const (
 	MB = 1024 * 1024
 	KB = 1024
+
+	BandwidthLimitModeClient = "client"
+	BandwidthLimitModeServer = "server"
 )
 
 type BandwidthQuantity struct {

pkg/config/visitor.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"reflect"
 
+	"github.com/samber/lo"
 	"gopkg.in/ini.v1"
 
 	"github.com/fatedier/frp/pkg/consts"
@@ -61,6 +62,11 @@ type STCPVisitorConf struct {
 
 type XTCPVisitorConf struct {
 	BaseVisitorConf `ini:",extends"`
+
+	Protocol         string `ini:"protocol" json:"protocol,omitempty"`
+	KeepTunnelOpen   bool   `ini:"keep_tunnel_open" json:"keep_tunnel_open,omitempty"`
+	MaxRetriesAnHour int    `ini:"max_retries_an_hour" json:"max_retries_an_hour,omitempty"`
+	MinRetryInterval int    `ini:"min_retry_interval" json:"min_retry_interval,omitempty"`
 }
 
 // DefaultVisitorConf creates a empty VisitorConf object by visitorType.
@@ -259,7 +265,12 @@ func (cfg *XTCPVisitorConf) Compare(cmp VisitorConf) bool {
 	}
 
 	// Add custom login equal, if exists
-
+	if cfg.Protocol != cmpConf.Protocol ||
+		cfg.KeepTunnelOpen != cmpConf.KeepTunnelOpen ||
+		cfg.MaxRetriesAnHour != cmpConf.MaxRetriesAnHour ||
+		cfg.MinRetryInterval != cmpConf.MinRetryInterval {
+		return false
+	}
 	return true
 }
 
@@ -270,7 +281,15 @@ func (cfg *XTCPVisitorConf) UnmarshalFromIni(prefix string, name string, section
 	}
 
 	// Add custom logic unmarshal, if exists
-
+	if cfg.Protocol == "" {
+		cfg.Protocol = "quic"
+	}
+	if cfg.MaxRetriesAnHour <= 0 {
+		cfg.MaxRetriesAnHour = 8
+	}
+	if cfg.MinRetryInterval <= 0 {
+		cfg.MinRetryInterval = 90
+	}
 	return
 }
 
@@ -280,6 +299,8 @@ func (cfg *XTCPVisitorConf) Check() (err error) {
 	}
 
 	// Add custom logic validate, if exists
-
+	if !lo.Contains([]string{"", "kcp", "quic"}, cfg.Protocol) {
+		return fmt.Errorf("protocol should be 'kcp' or 'quic'")
+	}
 	return
 }

pkg/config/visitor_test.go

@@ -87,6 +87,9 @@ func Test_Visitor_UnmarshalFromIni(t *testing.T) {
 					BindAddr:   "127.0.0.1",
 					BindPort:   9001,
 				},
+				Protocol:         "quic",
+				MaxRetriesAnHour: 8,
+				MinRetryInterval: 90,
 			},
 		},
 	}

pkg/metrics/mem/server.go

@@ -60,25 +60,30 @@ func (m *serverMetrics) run() {
 	go func() {
 		for {
 			time.Sleep(12 * time.Hour)
-			log.Debug("start to clear useless proxy statistics data...")
-			m.clearUselessInfo()
-			log.Debug("finish to clear useless proxy statistics data")
+			start := time.Now()
+			count, total := m.clearUselessInfo()
+			log.Debug("clear useless proxy statistics data count %d/%d, cost %v", count, total, time.Since(start))
 		}
 	}()
 }
 
-func (m *serverMetrics) clearUselessInfo() {
+func (m *serverMetrics) clearUselessInfo() (int, int) {
+	count := 0
+	total := 0
 	// To check if there are proxies that closed than 7 days and drop them.
 	m.mu.Lock()
 	defer m.mu.Unlock()
+	total = len(m.info.ProxyStatistics)
 	for name, data := range m.info.ProxyStatistics {
 		if !data.LastCloseTime.IsZero() &&
 			data.LastStartTime.Before(data.LastCloseTime) &&
 			time.Since(data.LastCloseTime) > time.Duration(7*24)*time.Hour {
 			delete(m.info.ProxyStatistics, name)
+			count++
 			log.Trace("clear proxy [%s]'s statistics data, lastCloseTime: [%s]", name, data.LastCloseTime.String())
 		}
 	}
+	return count, total
 }
 
 func (m *serverMetrics) NewClient() {

pkg/msg/ctl.go

@@ -42,3 +42,7 @@ func ReadMsgInto(c io.Reader, msg Message) (err error) {
 func WriteMsg(c io.Writer, msg interface{}) (err error) {
 	return msgCtl.WriteMsg(c, msg)
 }
+
+func Pack(msg interface{}) (data []byte, err error) {
+	return msgCtl.Pack(msg)
+}

pkg/msg/msg.go

@@ -14,50 +14,55 @@
 
 package msg
 
-import "net"
+import (
+	"net"
+	"reflect"
+)
 
 const (
-	TypeLogin                 = 'o'
-	TypeLoginResp             = '1'
-	TypeNewProxy              = 'p'
-	TypeNewProxyResp          = '2'
-	TypeCloseProxy            = 'c'
-	TypeNewWorkConn           = 'w'
-	TypeReqWorkConn           = 'r'
-	TypeStartWorkConn         = 's'
-	TypeNewVisitorConn        = 'v'
-	TypeNewVisitorConnResp    = '3'
-	TypePing                  = 'h'
-	TypePong                  = '4'
-	TypeUDPPacket             = 'u'
-	TypeNatHoleVisitor        = 'i'
-	TypeNatHoleClient         = 'n'
-	TypeNatHoleResp           = 'm'
-	TypeNatHoleClientDetectOK = 'd'
-	TypeNatHoleSid            = '5'
+	TypeLogin              = 'o'
+	TypeLoginResp          = '1'
+	TypeNewProxy           = 'p'
+	TypeNewProxyResp       = '2'
+	TypeCloseProxy         = 'c'
+	TypeNewWorkConn        = 'w'
+	TypeReqWorkConn        = 'r'
+	TypeStartWorkConn      = 's'
+	TypeNewVisitorConn     = 'v'
+	TypeNewVisitorConnResp = '3'
+	TypePing               = 'h'
+	TypePong               = '4'
+	TypeUDPPacket          = 'u'
+	TypeNatHoleVisitor     = 'i'
+	TypeNatHoleClient      = 'n'
+	TypeNatHoleResp        = 'm'
+	TypeNatHoleSid         = '5'
+	TypeNatHoleReport      = '6'
 )
 
 var msgTypeMap = map[byte]interface{}{
-	TypeLogin:                 Login{},
-	TypeLoginResp:             LoginResp{},
-	TypeNewProxy:              NewProxy{},
-	TypeNewProxyResp:          NewProxyResp{},
-	TypeCloseProxy:            CloseProxy{},
-	TypeNewWorkConn:           NewWorkConn{},
-	TypeReqWorkConn:           ReqWorkConn{},
-	TypeStartWorkConn:         StartWorkConn{},
-	TypeNewVisitorConn:        NewVisitorConn{},
-	TypeNewVisitorConnResp:    NewVisitorConnResp{},
-	TypePing:                  Ping{},
-	TypePong:                  Pong{},
-	TypeUDPPacket:             UDPPacket{},
-	TypeNatHoleVisitor:        NatHoleVisitor{},
-	TypeNatHoleClient:         NatHoleClient{},
-	TypeNatHoleResp:           NatHoleResp{},
-	TypeNatHoleClientDetectOK: NatHoleClientDetectOK{},
-	TypeNatHoleSid:            NatHoleSid{},
+	TypeLogin:              Login{},
+	TypeLoginResp:          LoginResp{},
+	TypeNewProxy:           NewProxy{},
+	TypeNewProxyResp:       NewProxyResp{},
+	TypeCloseProxy:         CloseProxy{},
+	TypeNewWorkConn:        NewWorkConn{},
+	TypeReqWorkConn:        ReqWorkConn{},
+	TypeStartWorkConn:      StartWorkConn{},
+	TypeNewVisitorConn:     NewVisitorConn{},
+	TypeNewVisitorConnResp: NewVisitorConnResp{},
+	TypePing:               Ping{},
+	TypePong:               Pong{},
+	TypeUDPPacket:          UDPPacket{},
+	TypeNatHoleVisitor:     NatHoleVisitor{},
+	TypeNatHoleClient:      NatHoleClient{},
+	TypeNatHoleResp:        NatHoleResp{},
+	TypeNatHoleSid:         NatHoleSid{},
+	TypeNatHoleReport:      NatHoleReport{},
 }
 
+var TypeNameNatHoleResp = reflect.TypeOf(&NatHoleResp{}).Elem().Name()
+
 // When frpc start, client send this message to login to server.
 type Login struct {
 	Version      string            `json:"version,omitempty"`
@@ -75,21 +80,22 @@ type Login struct {
 }
 
 type LoginResp struct {
-	Version       string `json:"version,omitempty"`
-	RunID         string `json:"run_id,omitempty"`
-	ServerUDPPort int    `json:"server_udp_port,omitempty"`
-	Error         string `json:"error,omitempty"`
+	Version string `json:"version,omitempty"`
+	RunID   string `json:"run_id,omitempty"`
+	Error   string `json:"error,omitempty"`
 }
 
 // When frpc login success, send this message to frps for running a new proxy.
 type NewProxy struct {
-	ProxyName      string            `json:"proxy_name,omitempty"`
-	ProxyType      string            `json:"proxy_type,omitempty"`
-	UseEncryption  bool              `json:"use_encryption,omitempty"`
-	UseCompression bool              `json:"use_compression,omitempty"`
-	Group          string            `json:"group,omitempty"`
-	GroupKey       string            `json:"group_key,omitempty"`
-	Metas          map[string]string `json:"metas,omitempty"`
+	ProxyName          string            `json:"proxy_name,omitempty"`
+	ProxyType          string            `json:"proxy_type,omitempty"`
+	UseEncryption      bool              `json:"use_encryption,omitempty"`
+	UseCompression     bool              `json:"use_compression,omitempty"`
+	BandwidthLimit     string            `json:"bandwidth_limit,omitempty"`
+	BandwidthLimitMode string            `json:"bandwidth_limit_mode,omitempty"`
+	Group              string            `json:"group,omitempty"`
+	GroupKey           string            `json:"group_key,omitempty"`
+	Metas              map[string]string `json:"metas,omitempty"`
 
 	// tcp and udp only
 	RemotePort int `json:"remote_port,omitempty"`
@@ -167,25 +173,58 @@ type UDPPacket struct {
 }
 
 type NatHoleVisitor struct {
-	ProxyName string `json:"proxy_name,omitempty"`
-	SignKey   string `json:"sign_key,omitempty"`
-	Timestamp int64  `json:"timestamp,omitempty"`
+	TransactionID string   `json:"transaction_id,omitempty"`
+	ProxyName     string   `json:"proxy_name,omitempty"`
+	PreCheck      bool     `json:"pre_check,omitempty"`
+	Protocol      string   `json:"protocol,omitempty"`
+	SignKey       string   `json:"sign_key,omitempty"`
+	Timestamp     int64    `json:"timestamp,omitempty"`
+	MappedAddrs   []string `json:"mapped_addrs,omitempty"`
+	AssistedAddrs []string `json:"assisted_addrs,omitempty"`
 }
 
 type NatHoleClient struct {
-	ProxyName string `json:"proxy_name,omitempty"`
-	Sid       string `json:"sid,omitempty"`
+	TransactionID string   `json:"transaction_id,omitempty"`
+	ProxyName     string   `json:"proxy_name,omitempty"`
+	Sid           string   `json:"sid,omitempty"`
+	MappedAddrs   []string `json:"mapped_addrs,omitempty"`
+	AssistedAddrs []string `json:"assisted_addrs,omitempty"`
+}
+
+type PortsRange struct {
+	From int `json:"from,omitempty"`
+	To   int `json:"to,omitempty"`
+}
+
+type NatHoleDetectBehavior struct {
+	Role              string       `json:"role,omitempty"` // sender or receiver
+	Mode              int          `json:"mode,omitempty"` // 0, 1, 2...
+	TTL               int          `json:"ttl,omitempty"`
+	SendDelayMs       int          `json:"send_delay_ms,omitempty"`
+	ReadTimeoutMs     int          `json:"read_timeout,omitempty"`
+	CandidatePorts    []PortsRange `json:"candidate_ports,omitempty"`
+	SendRandomPorts   int          `json:"send_random_ports,omitempty"`
+	ListenRandomPorts int          `json:"listen_random_ports,omitempty"`
 }
 
 type NatHoleResp struct {
-	Sid         string `json:"sid,omitempty"`
-	VisitorAddr string `json:"visitor_addr,omitempty"`
-	ClientAddr  string `json:"client_addr,omitempty"`
-	Error       string `json:"error,omitempty"`
+	TransactionID  string                `json:"transaction_id,omitempty"`
+	Sid            string                `json:"sid,omitempty"`
+	Protocol       string                `json:"protocol,omitempty"`
+	CandidateAddrs []string              `json:"candidate_addrs,omitempty"`
+	AssistedAddrs  []string              `json:"assisted_addrs,omitempty"`
+	DetectBehavior NatHoleDetectBehavior `json:"detect_behavior,omitempty"`
+	Error          string                `json:"error,omitempty"`
 }
 
-type NatHoleClientDetectOK struct{}
-
 type NatHoleSid struct {
-	Sid string `json:"sid,omitempty"`
+	TransactionID string `json:"transaction_id,omitempty"`
+	Sid           string `json:"sid,omitempty"`
+	Response      bool   `json:"response,omitempty"`
+	Nonce         string `json:"nonce,omitempty"`
+}
+
+type NatHoleReport struct {
+	Sid     string `json:"sid,omitempty"`
+	Success bool   `json:"success,omitempty"`
 }

pkg/nathole/analysis.go

@@ -0,0 +1,328 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nathole
+
+import (
+	"sync"
+	"time"
+
+	"github.com/samber/lo"
+)
+
+var (
+	// mode 0, both EasyNAT, PublicNetwork is always receiver
+	// sender | receiver, ttl 7
+	// receiver, ttl 7 | sender
+	// sender | receiver, ttl 4
+	// receiver, ttl 4 | sender
+	// sender | receiver
+	// receiver | sender
+	// sender, sendDelayMs 5000 | receiver
+	// sender, sendDelayMs 10000 | receiver
+	// receiver | sender, sendDelayMs 5000
+	// receiver | sender, sendDelayMs 10000
+	mode0Behaviors = []lo.Tuple2[RecommandBehavior, RecommandBehavior]{
+		lo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 7}),
+		lo.T2(RecommandBehavior{Role: DetectRoleReceiver, TTL: 7}, RecommandBehavior{Role: DetectRoleSender}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 4}),
+		lo.T2(RecommandBehavior{Role: DetectRoleReceiver, TTL: 4}, RecommandBehavior{Role: DetectRoleSender}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver}),
+		lo.T2(RecommandBehavior{Role: DetectRoleReceiver}, RecommandBehavior{Role: DetectRoleSender}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 5000}, RecommandBehavior{Role: DetectRoleReceiver}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 10000}, RecommandBehavior{Role: DetectRoleReceiver}),
+		lo.T2(RecommandBehavior{Role: DetectRoleReceiver}, RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 5000}),
+		lo.T2(RecommandBehavior{Role: DetectRoleReceiver}, RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 10000}),
+	}
+
+	// mode 1, HardNAT is sender, EasyNAT is receiver, port changes is regular
+	// sender | receiver, ttl 7, portsRangeNumber max 10
+	// sender, sendDelayMs 2000 | receiver, ttl 7, portsRangeNumber max 10
+	// sender | receiver, ttl 4, portsRangeNumber max 10
+	// sender, sendDelayMs 2000 | receiver, ttl 4, portsRangeNumber max 10
+	// sender | receiver, portsRangeNumber max 10
+	// sender, sendDelayMs 2000 | receiver, portsRangeNumber max 10
+	mode1Behaviors = []lo.Tuple2[RecommandBehavior, RecommandBehavior]{
+		lo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 7, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 2000}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 7, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 4, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 2000}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 4, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender}, RecommandBehavior{Role: DetectRoleReceiver, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender, SendDelayMs: 2000}, RecommandBehavior{Role: DetectRoleReceiver, PortsRangeNumber: 10}),
+	}
+
+	// mode 2, HardNAT is receiver, EasyNAT is sender
+	// sender, portsRandomNumber 1000, sendDelayMs 2000 | receiver, listen 256 ports, ttl 7
+	// sender, portsRandomNumber 1000, sendDelayMs 2000 | receiver, listen 256 ports, ttl 4
+	// sender, portsRandomNumber 1000, sendDelayMs 2000 | receiver, listen 256 ports
+	mode2Behaviors = []lo.Tuple2[RecommandBehavior, RecommandBehavior]{
+		lo.T2(
+			RecommandBehavior{Role: DetectRoleSender, PortsRandomNumber: 1000, SendDelayMs: 2000},
+			RecommandBehavior{Role: DetectRoleReceiver, ListenRandomPorts: 256, TTL: 7},
+		),
+		lo.T2(
+			RecommandBehavior{Role: DetectRoleSender, PortsRandomNumber: 1000, SendDelayMs: 2000},
+			RecommandBehavior{Role: DetectRoleReceiver, ListenRandomPorts: 256, TTL: 4},
+		),
+		lo.T2(
+			RecommandBehavior{Role: DetectRoleSender, PortsRandomNumber: 1000, SendDelayMs: 2000},
+			RecommandBehavior{Role: DetectRoleReceiver, ListenRandomPorts: 256},
+		),
+	}
+
+	// mode 3, For HardNAT & HardNAT, both changes in the ports are regular
+	// sender, portsRangeNumber 10 | receiver, ttl 7, portsRangeNumber 10
+	// sender, portsRangeNumber 10 | receiver, ttl 4, portsRangeNumber 10
+	// sender, portsRangeNumber 10 | receiver, portsRangeNumber 10
+	// receiver, ttl 7, portsRangeNumber 10 | sender, portsRangeNumber 10
+	// receiver, ttl 4, portsRangeNumber 10 | sender, portsRangeNumber 10
+	// receiver, portsRangeNumber 10 | sender, portsRangeNumber 10
+	mode3Behaviors = []lo.Tuple2[RecommandBehavior, RecommandBehavior]{
+		lo.T2(RecommandBehavior{Role: DetectRoleSender, PortsRangeNumber: 10}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 7, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender, PortsRangeNumber: 10}, RecommandBehavior{Role: DetectRoleReceiver, TTL: 4, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleSender, PortsRangeNumber: 10}, RecommandBehavior{Role: DetectRoleReceiver, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleReceiver, TTL: 7, PortsRangeNumber: 10}, RecommandBehavior{Role: DetectRoleSender, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleReceiver, TTL: 4, PortsRangeNumber: 10}, RecommandBehavior{Role: DetectRoleSender, PortsRangeNumber: 10}),
+		lo.T2(RecommandBehavior{Role: DetectRoleReceiver, PortsRangeNumber: 10}, RecommandBehavior{Role: DetectRoleSender, PortsRangeNumber: 10}),
+	}
+
+	// mode 4, Regular ports changes are usually the sender.
+	// sender, portsRandomNumber 1000, sendDelayMs: 2000 | receiver, listen 256 ports, ttl 7, portsRangeNumber 10
+	// sender, portsRandomNumber 1000, sendDelayMs: 2000 | receiver, listen 256 ports, ttl 4, portsRangeNumber 10
+	// sender, portsRandomNumber 1000, SendDelayMs: 2000 | receiver, listen 256 ports, portsRangeNumber 10
+	mode4Behaviors = []lo.Tuple2[RecommandBehavior, RecommandBehavior]{
+		lo.T2(
+			RecommandBehavior{Role: DetectRoleSender, PortsRandomNumber: 1000, SendDelayMs: 2000},
+			RecommandBehavior{Role: DetectRoleReceiver, ListenRandomPorts: 256, TTL: 7, PortsRangeNumber: 10},
+		),
+		lo.T2(
+			RecommandBehavior{Role: DetectRoleSender, PortsRandomNumber: 1000, SendDelayMs: 2000},
+			RecommandBehavior{Role: DetectRoleReceiver, ListenRandomPorts: 256, TTL: 4, PortsRangeNumber: 10},
+		),
+		lo.T2(
+			RecommandBehavior{Role: DetectRoleSender, PortsRandomNumber: 1000, SendDelayMs: 2000},
+			RecommandBehavior{Role: DetectRoleReceiver, ListenRandomPorts: 256, PortsRangeNumber: 10},
+		),
+	}
+)
+
+func getBehaviorByMode(mode int) []lo.Tuple2[RecommandBehavior, RecommandBehavior] {
+	switch mode {
+	case 0:
+		return mode0Behaviors
+	case 1:
+		return mode1Behaviors
+	case 2:
+		return mode2Behaviors
+	case 3:
+		return mode3Behaviors
+	case 4:
+		return mode4Behaviors
+	}
+	// default
+	return mode0Behaviors
+}
+
+func getBehaviorByModeAndIndex(mode int, index int) (RecommandBehavior, RecommandBehavior) {
+	behaviors := getBehaviorByMode(mode)
+	if index >= len(behaviors) {
+		return RecommandBehavior{}, RecommandBehavior{}
+	}
+	return behaviors[index].A, behaviors[index].B
+}
+
+func getBehaviorScoresByMode(mode int, defaultScore int) []*BehaviorScore {
+	return getBehaviorScoresByMode2(mode, defaultScore, defaultScore)
+}
+
+func getBehaviorScoresByMode2(mode int, senderScore, receiverScore int) []*BehaviorScore {
+	behaviors := getBehaviorByMode(mode)
+	scores := make([]*BehaviorScore, 0, len(behaviors))
+	for i := 0; i < len(behaviors); i++ {
+		score := receiverScore
+		if behaviors[i].A.Role == DetectRoleSender {
+			score = senderScore
+		}
+		scores = append(scores, &BehaviorScore{Mode: mode, Index: i, Score: score})
+	}
+	return scores
+}
+
+type RecommandBehavior struct {
+	Role              string
+	TTL               int
+	SendDelayMs       int
+	PortsRangeNumber  int
+	PortsRandomNumber int
+	ListenRandomPorts int
+}
+
+type MakeHoleRecords struct {
+	mu             sync.Mutex
+	scores         []*BehaviorScore
+	LastUpdateTime time.Time
+}
+
+func NewMakeHoleRecords(c, v *NatFeature) *MakeHoleRecords {
+	scores := []*BehaviorScore{}
+	easyCount, hardCount, portsChangedRegularCount := ClassifyFeatureCount([]*NatFeature{c, v})
+	appendMode0 := func() {
+		switch {
+		case c.PublicNetwork:
+			scores = append(scores, getBehaviorScoresByMode2(DetectMode0, 0, 1)...)
+		case v.PublicNetwork:
+			scores = append(scores, getBehaviorScoresByMode2(DetectMode0, 1, 0)...)
+		default:
+			scores = append(scores, getBehaviorScoresByMode(DetectMode0, 0)...)
+		}
+	}
+
+	switch {
+	case easyCount == 2:
+		appendMode0()
+	case hardCount == 1 && portsChangedRegularCount == 1:
+		scores = append(scores, getBehaviorScoresByMode(DetectMode1, 0)...)
+		scores = append(scores, getBehaviorScoresByMode(DetectMode2, 0)...)
+		appendMode0()
+	case hardCount == 1 && portsChangedRegularCount == 0:
+		scores = append(scores, getBehaviorScoresByMode(DetectMode2, 0)...)
+		scores = append(scores, getBehaviorScoresByMode(DetectMode1, 0)...)
+		appendMode0()
+	case hardCount == 2 && portsChangedRegularCount == 2:
+		scores = append(scores, getBehaviorScoresByMode(DetectMode3, 0)...)
+		scores = append(scores, getBehaviorScoresByMode(DetectMode4, 0)...)
+	case hardCount == 2 && portsChangedRegularCount == 1:
+		scores = append(scores, getBehaviorScoresByMode(DetectMode4, 0)...)
+	default:
+		// hard to make hole, just trying it out.
+		scores = append(scores, getBehaviorScoresByMode(DetectMode0, 1)...)
+		scores = append(scores, getBehaviorScoresByMode(DetectMode1, 1)...)
+		scores = append(scores, getBehaviorScoresByMode(DetectMode3, 1)...)
+	}
+	return &MakeHoleRecords{scores: scores, LastUpdateTime: time.Now()}
+}
+
+func (mhr *MakeHoleRecords) ReportSuccess(mode int, index int) {
+	mhr.mu.Lock()
+	defer mhr.mu.Unlock()
+	mhr.LastUpdateTime = time.Now()
+	for i := range mhr.scores {
+		score := mhr.scores[i]
+		if score.Mode != mode || score.Index != index {
+			continue
+		}
+
+		score.Score += 2
+		score.Score = lo.Min([]int{score.Score, 10})
+		return
+	}
+}
+
+func (mhr *MakeHoleRecords) Recommand() (mode, index int) {
+	mhr.mu.Lock()
+	defer mhr.mu.Unlock()
+
+	maxScore := lo.MaxBy(mhr.scores, func(item, max *BehaviorScore) bool {
+		return item.Score > max.Score
+	})
+	if maxScore == nil {
+		return 0, 0
+	}
+	maxScore.Score--
+	mhr.LastUpdateTime = time.Now()
+	return maxScore.Mode, maxScore.Index
+}
+
+type BehaviorScore struct {
+	Mode  int
+	Index int
+	// between -10 and 10
+	Score int
+}
+
+type Analyzer struct {
+	// key is client ip + visitor ip
+	records             map[string]*MakeHoleRecords
+	dataReserveDuration time.Duration
+
+	mu sync.Mutex
+}
+
+func NewAnalyzer(dataReserveDuration time.Duration) *Analyzer {
+	return &Analyzer{
+		records:             make(map[string]*MakeHoleRecords),
+		dataReserveDuration: dataReserveDuration,
+	}
+}
+
+func (a *Analyzer) GetRecommandBehaviors(key string, c, v *NatFeature) (mode, index int, _ RecommandBehavior, _ RecommandBehavior) {
+	a.mu.Lock()
+	records, ok := a.records[key]
+	if !ok {
+		records = NewMakeHoleRecords(c, v)
+		a.records[key] = records
+	}
+	a.mu.Unlock()
+
+	mode, index = records.Recommand()
+	cBehavior, vBehavior := getBehaviorByModeAndIndex(mode, index)
+
+	switch mode {
+	case DetectMode1:
+		// HardNAT is always the sender
+		if c.NatType == EasyNAT {
+			cBehavior, vBehavior = vBehavior, cBehavior
+		}
+	case DetectMode2:
+		// HardNAT is always the receiver
+		if c.NatType == HardNAT {
+			cBehavior, vBehavior = vBehavior, cBehavior
+		}
+	case DetectMode4:
+		// Regular ports changes is always the sender
+		if !c.RegularPortsChange {
+			cBehavior, vBehavior = vBehavior, cBehavior
+		}
+	}
+	return mode, index, cBehavior, vBehavior
+}
+
+func (a *Analyzer) ReportSuccess(key string, mode, index int) {
+	a.mu.Lock()
+	records, ok := a.records[key]
+	a.mu.Unlock()
+	if !ok {
+		return
+	}
+	records.ReportSuccess(mode, index)
+}
+
+func (a *Analyzer) Clean() (int, int) {
+	now := time.Now()
+	total := 0
+	count := 0
+
+	// cleanup 10w records may take 5ms
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	total = len(a.records)
+	// clean up records that have not been used for a period of time.
+	for key, records := range a.records {
+		if now.Sub(records.LastUpdateTime) > a.dataReserveDuration {
+			delete(a.records, key)
+			count++
+		}
+	}
+	return count, total
+}

pkg/nathole/classify.go

@@ -0,0 +1,127 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nathole
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+
+	"github.com/samber/lo"
+)
+
+const (
+	EasyNAT = "EasyNAT"
+	HardNAT = "HardNAT"
+
+	BehaviorNoChange    = "BehaviorNoChange"
+	BehaviorIPChanged   = "BehaviorIPChanged"
+	BehaviorPortChanged = "BehaviorPortChanged"
+	BehaviorBothChanged = "BehaviorBothChanged"
+)
+
+type NatFeature struct {
+	NatType            string
+	Behavior           string
+	PortsDifference    int
+	RegularPortsChange bool
+	PublicNetwork      bool
+}
+
+func ClassifyNATFeature(addresses []string, localIPs []string) (*NatFeature, error) {
+	if len(addresses) <= 1 {
+		return nil, fmt.Errorf("not enough addresses")
+	}
+	natFeature := &NatFeature{}
+	ipChanged := false
+	portChanged := false
+
+	var baseIP, basePort string
+	var portMax, portMin int
+	for _, addr := range addresses {
+		ip, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		portNum, err := strconv.Atoi(port)
+		if err != nil {
+			return nil, err
+		}
+		if lo.Contains(localIPs, ip) {
+			natFeature.PublicNetwork = true
+		}
+
+		if baseIP == "" {
+			baseIP = ip
+			basePort = port
+			portMax = portNum
+			portMin = portNum
+			continue
+		}
+
+		if portNum > portMax {
+			portMax = portNum
+		}
+		if portNum < portMin {
+			portMin = portNum
+		}
+		if baseIP != ip {
+			ipChanged = true
+		}
+		if basePort != port {
+			portChanged = true
+		}
+	}
+
+	natFeature.PortsDifference = portMax - portMin
+	if natFeature.PortsDifference <= 10 && natFeature.PortsDifference >= 1 {
+		natFeature.RegularPortsChange = true
+	}
+
+	switch {
+	case ipChanged && portChanged:
+		natFeature.NatType = HardNAT
+		natFeature.Behavior = BehaviorBothChanged
+	case ipChanged:
+		natFeature.NatType = HardNAT
+		natFeature.Behavior = BehaviorIPChanged
+	case portChanged:
+		natFeature.NatType = HardNAT
+		natFeature.Behavior = BehaviorPortChanged
+	default:
+		natFeature.NatType = EasyNAT
+		natFeature.Behavior = BehaviorNoChange
+	}
+	return natFeature, nil
+}
+
+func ClassifyFeatureCount(features []*NatFeature) (int, int, int) {
+	easyCount := 0
+	hardCount := 0
+	// for HardNAT
+	portsChangedRegularCount := 0
+	for _, feature := range features {
+		if feature.NatType == EasyNAT {
+			easyCount++
+			continue
+		}
+
+		hardCount++
+		if feature.RegularPortsChange {
+			portsChangedRegularCount++
+		}
+	}
+	return easyCount, hardCount, portsChangedRegularCount
+}

pkg/nathole/controller.go

@@ -0,0 +1,382 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nathole
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/fatedier/golib/errors"
+	"github.com/samber/lo"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/fatedier/frp/pkg/msg"
+	"github.com/fatedier/frp/pkg/transport"
+	"github.com/fatedier/frp/pkg/util/log"
+	"github.com/fatedier/frp/pkg/util/util"
+)
+
+// NatHoleTimeout seconds.
+var NatHoleTimeout int64 = 10
+
+func NewTransactionID() string {
+	id, _ := util.RandID()
+	return fmt.Sprintf("%d%s", time.Now().Unix(), id)
+}
+
+type ClientCfg struct {
+	name  string
+	sk    string
+	sidCh chan string
+}
+
+type Session struct {
+	sid            string
+	analysisKey    string
+	recommandMode  int
+	recommandIndex int
+
+	visitorMsg         *msg.NatHoleVisitor
+	visitorTransporter transport.MessageTransporter
+	vResp              *msg.NatHoleResp
+	vNatFeature        *NatFeature
+	vBehavior          RecommandBehavior
+
+	clientMsg         *msg.NatHoleClient
+	clientTransporter transport.MessageTransporter
+	cResp             *msg.NatHoleResp
+	cNatFeature       *NatFeature
+	cBehavior         RecommandBehavior
+
+	notifyCh chan struct{}
+}
+
+func (s *Session) genAnalysisKey() {
+	hash := md5.New()
+	vIPs := lo.Uniq(parseIPs(s.visitorMsg.MappedAddrs))
+	if len(vIPs) > 0 {
+		hash.Write([]byte(vIPs[0]))
+	}
+	hash.Write([]byte(s.vNatFeature.NatType))
+	hash.Write([]byte(s.vNatFeature.Behavior))
+	hash.Write([]byte(strconv.FormatBool(s.vNatFeature.RegularPortsChange)))
+
+	cIPs := lo.Uniq(parseIPs(s.clientMsg.MappedAddrs))
+	if len(cIPs) > 0 {
+		hash.Write([]byte(cIPs[0]))
+	}
+	hash.Write([]byte(s.cNatFeature.NatType))
+	hash.Write([]byte(s.cNatFeature.Behavior))
+	hash.Write([]byte(strconv.FormatBool(s.cNatFeature.RegularPortsChange)))
+	s.analysisKey = hex.EncodeToString(hash.Sum(nil))
+}
+
+type Controller struct {
+	clientCfgs map[string]*ClientCfg
+	sessions   map[string]*Session
+	analyzer   *Analyzer
+
+	mu sync.RWMutex
+}
+
+func NewController(analysisDataReserveDuration time.Duration) (*Controller, error) {
+	return &Controller{
+		clientCfgs: make(map[string]*ClientCfg),
+		sessions:   make(map[string]*Session),
+		analyzer:   NewAnalyzer(analysisDataReserveDuration),
+	}, nil
+}
+
+func (c *Controller) CleanWorker(ctx context.Context) {
+	ticker := time.NewTicker(time.Hour)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+			start := time.Now()
+			count, total := c.analyzer.Clean()
+			log.Trace("clean %d/%d nathole analysis data, cost %v", count, total, time.Since(start))
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (c *Controller) ListenClient(name string, sk string) chan string {
+	cfg := &ClientCfg{
+		name:  name,
+		sk:    sk,
+		sidCh: make(chan string),
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.clientCfgs[name] = cfg
+	return cfg.sidCh
+}
+
+func (c *Controller) CloseClient(name string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.clientCfgs, name)
+}
+
+func (c *Controller) GenSid() string {
+	t := time.Now().Unix()
+	id, _ := util.RandID()
+	return fmt.Sprintf("%d%s", t, id)
+}
+
+func (c *Controller) HandleVisitor(m *msg.NatHoleVisitor, transporter transport.MessageTransporter) {
+	if m.PreCheck {
+		_, ok := c.clientCfgs[m.ProxyName]
+		if !ok {
+			_ = transporter.Send(c.GenNatHoleResponse(m.TransactionID, nil, fmt.Sprintf("xtcp server for [%s] doesn't exist", m.ProxyName)))
+		} else {
+			_ = transporter.Send(c.GenNatHoleResponse(m.TransactionID, nil, ""))
+		}
+		return
+	}
+
+	sid := c.GenSid()
+	session := &Session{
+		sid:                sid,
+		visitorMsg:         m,
+		visitorTransporter: transporter,
+		notifyCh:           make(chan struct{}, 1),
+	}
+	var (
+		clientCfg *ClientCfg
+		ok        bool
+	)
+	err := func() error {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+
+		clientCfg, ok = c.clientCfgs[m.ProxyName]
+		if !ok {
+			return fmt.Errorf("xtcp server for [%s] doesn't exist", m.ProxyName)
+		}
+		if !util.ConstantTimeEqString(m.SignKey, util.GetAuthKey(clientCfg.sk, m.Timestamp)) {
+			return fmt.Errorf("xtcp connection of [%s] auth failed", m.ProxyName)
+		}
+		c.sessions[sid] = session
+		return nil
+	}()
+	if err != nil {
+		log.Warn("handle visitorMsg error: %v", err)
+		_ = transporter.Send(c.GenNatHoleResponse(m.TransactionID, nil, err.Error()))
+		return
+	}
+	log.Trace("handle visitor message, sid [%s]", sid)
+
+	defer func() {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		delete(c.sessions, sid)
+	}()
+
+	if err := errors.PanicToError(func() {
+		clientCfg.sidCh <- sid
+	}); err != nil {
+		return
+	}
+
+	// wait for NatHoleClient message
+	select {
+	case <-session.notifyCh:
+	case <-time.After(time.Duration(NatHoleTimeout) * time.Second):
+		log.Debug("wait for NatHoleClient message timeout, sid [%s]", sid)
+		return
+	}
+
+	// Make hole-punching decisions based on the NAT information of the client and visitor.
+	vResp, cResp, err := c.analysis(session)
+	if err != nil {
+		log.Debug("sid [%s] analysis error: %v", err)
+		vResp = c.GenNatHoleResponse(session.visitorMsg.TransactionID, nil, err.Error())
+		cResp = c.GenNatHoleResponse(session.clientMsg.TransactionID, nil, err.Error())
+	}
+	session.cResp = cResp
+	session.vResp = vResp
+
+	// send response to visitor and client
+	var g errgroup.Group
+	g.Go(func() error {
+		// if it's sender, wait for a while to make sure the client has send the detect messages
+		if vResp.DetectBehavior.Role == "sender" {
+			time.Sleep(1 * time.Second)
+		}
+		_ = session.visitorTransporter.Send(vResp)
+		return nil
+	})
+	g.Go(func() error {
+		// if it's sender, wait for a while to make sure the client has send the detect messages
+		if cResp.DetectBehavior.Role == "sender" {
+			time.Sleep(1 * time.Second)
+		}
+		_ = session.clientTransporter.Send(cResp)
+		return nil
+	})
+	_ = g.Wait()
+
+	time.Sleep(time.Duration(cResp.DetectBehavior.ReadTimeoutMs+30000) * time.Millisecond)
+}
+
+func (c *Controller) HandleClient(m *msg.NatHoleClient, transporter transport.MessageTransporter) {
+	c.mu.RLock()
+	session, ok := c.sessions[m.Sid]
+	c.mu.RUnlock()
+	if !ok {
+		return
+	}
+	log.Trace("handle client message, sid [%s]", session.sid)
+	session.clientMsg = m
+	session.clientTransporter = transporter
+	select {
+	case session.notifyCh <- struct{}{}:
+	default:
+	}
+}
+
+func (c *Controller) HandleReport(m *msg.NatHoleReport) {
+	c.mu.RLock()
+	session, ok := c.sessions[m.Sid]
+	c.mu.RUnlock()
+	if !ok {
+		log.Trace("sid [%s] report make hole success: %v, but session not found", m.Sid, m.Success)
+		return
+	}
+	if m.Success {
+		c.analyzer.ReportSuccess(session.analysisKey, session.recommandMode, session.recommandIndex)
+	}
+	log.Info("sid [%s] report make hole success: %v, mode %v, index %v",
+		m.Sid, m.Success, session.recommandMode, session.recommandIndex)
+}
+
+func (c *Controller) GenNatHoleResponse(transactionID string, session *Session, errInfo string) *msg.NatHoleResp {
+	var sid string
+	if session != nil {
+		sid = session.sid
+	}
+	return &msg.NatHoleResp{
+		TransactionID: transactionID,
+		Sid:           sid,
+		Error:         errInfo,
+	}
+}
+
+// analysis analyzes the NAT type and behavior of the visitor and client, then makes hole-punching decisions.
+// return the response to the visitor and client.
+func (c *Controller) analysis(session *Session) (*msg.NatHoleResp, *msg.NatHoleResp, error) {
+	cm := session.clientMsg
+	vm := session.visitorMsg
+
+	cNatFeature, err := ClassifyNATFeature(cm.MappedAddrs, parseIPs(cm.AssistedAddrs))
+	if err != nil {
+		return nil, nil, fmt.Errorf("classify client nat feature error: %v", err)
+	}
+
+	vNatFeature, err := ClassifyNATFeature(vm.MappedAddrs, parseIPs(vm.AssistedAddrs))
+	if err != nil {
+		return nil, nil, fmt.Errorf("classify visitor nat feature error: %v", err)
+	}
+	session.cNatFeature = cNatFeature
+	session.vNatFeature = vNatFeature
+	session.genAnalysisKey()
+
+	mode, index, cBehavior, vBehavior := c.analyzer.GetRecommandBehaviors(session.analysisKey, cNatFeature, vNatFeature)
+	session.recommandMode = mode
+	session.recommandIndex = index
+	session.cBehavior = cBehavior
+	session.vBehavior = vBehavior
+
+	timeoutMs := lo.Max([]int{cBehavior.SendDelayMs, vBehavior.SendDelayMs}) + 5000
+	if cBehavior.ListenRandomPorts > 0 || vBehavior.ListenRandomPorts > 0 {
+		timeoutMs += 30000
+	}
+
+	protocol := vm.Protocol
+	vResp := &msg.NatHoleResp{
+		TransactionID:  vm.TransactionID,
+		Sid:            session.sid,
+		Protocol:       protocol,
+		CandidateAddrs: lo.Uniq(cm.MappedAddrs),
+		AssistedAddrs:  lo.Uniq(cm.AssistedAddrs),
+		DetectBehavior: msg.NatHoleDetectBehavior{
+			Mode:              mode,
+			Role:              vBehavior.Role,
+			TTL:               vBehavior.TTL,
+			SendDelayMs:       vBehavior.SendDelayMs,
+			ReadTimeoutMs:     timeoutMs - vBehavior.SendDelayMs,
+			SendRandomPorts:   vBehavior.PortsRandomNumber,
+			ListenRandomPorts: vBehavior.ListenRandomPorts,
+			CandidatePorts:    getRangePorts(cm.MappedAddrs, cNatFeature.PortsDifference, vBehavior.PortsRangeNumber),
+		},
+	}
+	cResp := &msg.NatHoleResp{
+		TransactionID:  cm.TransactionID,
+		Sid:            session.sid,
+		Protocol:       protocol,
+		CandidateAddrs: lo.Uniq(vm.MappedAddrs),
+		AssistedAddrs:  lo.Uniq(vm.AssistedAddrs),
+		DetectBehavior: msg.NatHoleDetectBehavior{
+			Mode:              mode,
+			Role:              cBehavior.Role,
+			TTL:               cBehavior.TTL,
+			SendDelayMs:       cBehavior.SendDelayMs,
+			ReadTimeoutMs:     timeoutMs - cBehavior.SendDelayMs,
+			SendRandomPorts:   cBehavior.PortsRandomNumber,
+			ListenRandomPorts: cBehavior.ListenRandomPorts,
+			CandidatePorts:    getRangePorts(vm.MappedAddrs, vNatFeature.PortsDifference, cBehavior.PortsRangeNumber),
+		},
+	}
+
+	log.Debug("sid [%s] visitor nat: %+v, candidateAddrs: %v; client nat: %+v, candidateAddrs: %v, protocol: %s",
+		session.sid, *vNatFeature, vm.MappedAddrs, *cNatFeature, cm.MappedAddrs, protocol)
+	log.Debug("sid [%s] visitor detect behavior: %+v", session.sid, vResp.DetectBehavior)
+	log.Debug("sid [%s] client detect behavior: %+v", session.sid, cResp.DetectBehavior)
+	return vResp, cResp, nil
+}
+
+func getRangePorts(addrs []string, difference, maxNumber int) []msg.PortsRange {
+	if maxNumber <= 0 {
+		return nil
+	}
+
+	addr, err := lo.Last(addrs)
+	if err != nil {
+		return nil
+	}
+	var ports []msg.PortsRange
+	_, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return nil
+	}
+	ports = append(ports, msg.PortsRange{
+		From: lo.Max([]int{port - difference - 5, port - maxNumber, 1}),
+		To:   lo.Min([]int{port + difference + 5, port + maxNumber, 65535}),
+	})
+	return ports
+}

pkg/nathole/discovery.go

@@ -0,0 +1,185 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nathole
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/pion/stun"
+)
+
+var responseTimeout = 3 * time.Second
+
+type Message struct {
+	Body []byte
+	Addr string
+}
+
+// If the localAddr is empty, it will listen on a random port.
+func Discover(stunServers []string, localAddr string) ([]string, net.Addr, error) {
+	// create a discoverConn and get response from messageChan
+	discoverConn, err := listen(localAddr)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer discoverConn.Close()
+
+	go discoverConn.readLoop()
+
+	addresses := make([]string, 0, len(stunServers))
+	for _, addr := range stunServers {
+		// get external address from stun server
+		externalAddrs, err := discoverConn.discoverFromStunServer(addr)
+		if err != nil {
+			return nil, nil, err
+		}
+		addresses = append(addresses, externalAddrs...)
+	}
+	return addresses, discoverConn.localAddr, nil
+}
+
+type stunResponse struct {
+	externalAddr string
+	otherAddr    string
+}
+
+type discoverConn struct {
+	conn *net.UDPConn
+
+	localAddr   net.Addr
+	messageChan chan *Message
+}
+
+func listen(localAddr string) (*discoverConn, error) {
+	var local *net.UDPAddr
+	if localAddr != "" {
+		addr, err := net.ResolveUDPAddr("udp4", localAddr)
+		if err != nil {
+			return nil, err
+		}
+		local = addr
+	}
+	conn, err := net.ListenUDP("udp4", local)
+	if err != nil {
+		return nil, err
+	}
+
+	return &discoverConn{
+		conn:        conn,
+		localAddr:   conn.LocalAddr(),
+		messageChan: make(chan *Message, 10),
+	}, nil
+}
+
+func (c *discoverConn) Close() error {
+	if c.messageChan != nil {
+		close(c.messageChan)
+		c.messageChan = nil
+	}
+	return c.conn.Close()
+}
+
+func (c *discoverConn) readLoop() {
+	for {
+		buf := make([]byte, 1024)
+		n, addr, err := c.conn.ReadFromUDP(buf)
+		if err != nil {
+			return
+		}
+		buf = buf[:n]
+
+		c.messageChan <- &Message{
+			Body: buf,
+			Addr: addr.String(),
+		}
+	}
+}
+
+func (c *discoverConn) doSTUNRequest(addr string) (*stunResponse, error) {
+	serverAddr, err := net.ResolveUDPAddr("udp4", addr)
+	if err != nil {
+		return nil, err
+	}
+	request, err := stun.Build(stun.TransactionID, stun.BindingRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = request.NewTransactionID(); err != nil {
+		return nil, err
+	}
+	if _, err := c.conn.WriteTo(request.Raw, serverAddr); err != nil {
+		return nil, err
+	}
+
+	var m stun.Message
+	select {
+	case msg := <-c.messageChan:
+		m.Raw = msg.Body
+		if err := m.Decode(); err != nil {
+			return nil, err
+		}
+	case <-time.After(responseTimeout):
+		return nil, fmt.Errorf("wait response from stun server timeout")
+	}
+	xorAddrGetter := &stun.XORMappedAddress{}
+	mappedAddrGetter := &stun.MappedAddress{}
+	changedAddrGetter := ChangedAddress{}
+	otherAddrGetter := &stun.OtherAddress{}
+
+	resp := &stunResponse{}
+	if err := mappedAddrGetter.GetFrom(&m); err == nil {
+		resp.externalAddr = mappedAddrGetter.String()
+	}
+	if err := xorAddrGetter.GetFrom(&m); err == nil {
+		resp.externalAddr = xorAddrGetter.String()
+	}
+	if err := changedAddrGetter.GetFrom(&m); err == nil {
+		resp.otherAddr = changedAddrGetter.String()
+	}
+	if err := otherAddrGetter.GetFrom(&m); err == nil {
+		resp.otherAddr = otherAddrGetter.String()
+	}
+	return resp, nil
+}
+
+func (c *discoverConn) discoverFromStunServer(addr string) ([]string, error) {
+	resp, err := c.doSTUNRequest(addr)
+	if err != nil {
+		return nil, err
+	}
+	if resp.externalAddr == "" {
+		return nil, fmt.Errorf("no external address found")
+	}
+
+	externalAddrs := make([]string, 0, 2)
+	externalAddrs = append(externalAddrs, resp.externalAddr)
+
+	if resp.otherAddr == "" {
+		return externalAddrs, nil
+	}
+
+	// find external address from changed address
+	resp, err = c.doSTUNRequest(resp.otherAddr)
+	if err != nil {
+		return nil, err
+	}
+	if resp.externalAddr != "" {
+		externalAddrs = append(externalAddrs, resp.externalAddr)
+	}
+	return externalAddrs, nil
+}

pkg/nathole/nathole.go

@@ -1,212 +1,440 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package nathole
 
 import (
-	"bytes"
+	"context"
 	"fmt"
+	"math/rand"
 	"net"
-	"sync"
+	"strconv"
+	"strings"
 	"time"
 
-	"github.com/fatedier/golib/errors"
 	"github.com/fatedier/golib/pool"
+	"github.com/samber/lo"
+	"golang.org/x/net/ipv4"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/fatedier/frp/pkg/msg"
-	"github.com/fatedier/frp/pkg/util/log"
-	"github.com/fatedier/frp/pkg/util/util"
+	"github.com/fatedier/frp/pkg/transport"
+	"github.com/fatedier/frp/pkg/util/xlog"
 )
 
-// NatHoleTimeout seconds.
-var NatHoleTimeout int64 = 10
+var (
+	// mode 0: simple detect mode, usually for both EasyNAT or HardNAT & EasyNAT(Public Network)
+	// a. receiver sends detect message with low TTL
+	// b. sender sends normal detect message to receiver
+	// c. receiver receives detect message and sends back a message to sender
+	//
+	// mode 1: For HardNAT & EasyNAT, send detect messages to multiple guessed ports.
+	// Usually applicable to scenarios where port changes are regular.
+	// Most of the steps are the same as mode 0, but EasyNAT is fixed as the receiver and will send detect messages
+	// with low TTL to multiple guessed ports of the sender.
+	//
+	// mode 2: For HardNAT & EasyNAT, ports changes are not regular.
+	// a. HardNAT machine will listen on multiple ports and send detect messages with low TTL to EasyNAT machine
+	// b. EasyNAT machine will send detect messages to random ports of HardNAT machine.
+	//
+	// mode 3: For HardNAT & HardNAT, both changes in the ports are regular.
+	// Most of the steps are the same as mode 1, but the sender also needs to send detect messages to multiple guessed
+	// ports of the receiver.
+	//
+	// mode 4: For HardNAT & HardNAT, one of the changes in the ports is regular.
+	// Regular port changes are usually on the sender side.
+	// a. Receiver listens on multiple ports and sends detect messages with low TTL to the sender's guessed range ports.
+	// b. Sender sends detect messages to random ports of the receiver.
+	SupportedModes = []int{DetectMode0, DetectMode1, DetectMode2, DetectMode3, DetectMode4}
+	SupportedRoles = []string{DetectRoleSender, DetectRoleReceiver}
 
-type SidRequest struct {
-	Sid      string
-	NotifyCh chan struct{}
+	DetectMode0        = 0
+	DetectMode1        = 1
+	DetectMode2        = 2
+	DetectMode3        = 3
+	DetectMode4        = 4
+	DetectRoleSender   = "sender"
+	DetectRoleReceiver = "receiver"
+)
+
+type PrepareResult struct {
+	Addrs         []string
+	AssistedAddrs []string
+	ListenConn    *net.UDPConn
+	NatType       string
+	Behavior      string
 }
 
-type Controller struct {
-	listener *net.UDPConn
+// PreCheck is used to check if the proxy is ready for penetration.
+// Call this function before calling Prepare to avoid unnecessary preparation work.
+func PreCheck(
+	ctx context.Context, transporter transport.MessageTransporter,
+	proxyName string, timeout time.Duration,
+) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
 
-	clientCfgs map[string]*ClientCfg
-	sessions   map[string]*Session
-
-	mu sync.RWMutex
-}
-
-func NewController(udpBindAddr string) (nc *Controller, err error) {
-	addr, err := net.ResolveUDPAddr("udp", udpBindAddr)
+	var natHoleRespMsg *msg.NatHoleResp
+	transactionID := NewTransactionID()
+	m, err := transporter.Do(timeoutCtx, &msg.NatHoleVisitor{
+		TransactionID: transactionID,
+		ProxyName:     proxyName,
+		PreCheck:      true,
+	}, transactionID, msg.TypeNameNatHoleResp)
 	if err != nil {
-		return nil, err
+		return fmt.Errorf("get natHoleRespMsg error: %v", err)
 	}
-	lconn, err := net.ListenUDP("udp", addr)
+	mm, ok := m.(*msg.NatHoleResp)
+	if !ok {
+		return fmt.Errorf("get natHoleRespMsg error: invalid message type")
+	}
+	natHoleRespMsg = mm
+
+	if natHoleRespMsg.Error != "" {
+		return fmt.Errorf("%s", natHoleRespMsg.Error)
+	}
+	return nil
+}
+
+// Prepare is used to do some preparation work before penetration.
+func Prepare(stunServers []string) (*PrepareResult, error) {
+	// discover for Nat type
+	addrs, localAddr, err := Discover(stunServers, "")
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("discover error: %v", err)
 	}
-	nc = &Controller{
-		listener:   lconn,
-		clientCfgs: make(map[string]*ClientCfg),
-		sessions:   make(map[string]*Session),
+	if len(addrs) < 2 {
+		return nil, fmt.Errorf("discover error: not enough addresses")
 	}
-	return nc, nil
+
+	localIPs, _ := ListLocalIPsForNatHole(10)
+	natFeature, err := ClassifyNATFeature(addrs, localIPs)
+	if err != nil {
+		return nil, fmt.Errorf("classify nat feature error: %v", err)
+	}
+
+	laddr, err := net.ResolveUDPAddr("udp4", localAddr.String())
+	if err != nil {
+		return nil, fmt.Errorf("resolve local udp addr error: %v", err)
+	}
+	listenConn, err := net.ListenUDP("udp4", laddr)
+	if err != nil {
+		return nil, fmt.Errorf("listen local udp addr error: %v", err)
+	}
+
+	assistedAddrs := make([]string, 0, len(localIPs))
+	for _, ip := range localIPs {
+		assistedAddrs = append(assistedAddrs, net.JoinHostPort(ip, strconv.Itoa(laddr.Port)))
+	}
+	return &PrepareResult{
+		Addrs:         addrs,
+		AssistedAddrs: assistedAddrs,
+		ListenConn:    listenConn,
+		NatType:       natFeature.NatType,
+		Behavior:      natFeature.Behavior,
+	}, nil
 }
 
-func (nc *Controller) ListenClient(name string, sk string) (sidCh chan *SidRequest) {
-	clientCfg := &ClientCfg{
-		Name:  name,
-		Sk:    sk,
-		SidCh: make(chan *SidRequest),
+// ExchangeInfo is used to exchange information between client and visitor.
+// 1. Send input message to server by msgTransporter.
+// 2. Server will gather information from client and visitor and analyze it. Then send back a NatHoleResp message to them to tell them how to do next.
+// 3. Receive NatHoleResp message from server.
+func ExchangeInfo(
+	ctx context.Context, transporter transport.MessageTransporter,
+	laneKey string, m msg.Message, timeout time.Duration,
+) (*msg.NatHoleResp, error) {
+	timeoutCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	var natHoleRespMsg *msg.NatHoleResp
+	m, err := transporter.Do(timeoutCtx, m, laneKey, msg.TypeNameNatHoleResp)
+	if err != nil {
+		return nil, fmt.Errorf("get natHoleRespMsg error: %v", err)
 	}
-	nc.mu.Lock()
-	nc.clientCfgs[name] = clientCfg
-	nc.mu.Unlock()
-	return clientCfg.SidCh
+	mm, ok := m.(*msg.NatHoleResp)
+	if !ok {
+		return nil, fmt.Errorf("get natHoleRespMsg error: invalid message type")
+	}
+	natHoleRespMsg = mm
+
+	if natHoleRespMsg.Error != "" {
+		return nil, fmt.Errorf("natHoleRespMsg get error info: %s", natHoleRespMsg.Error)
+	}
+	if len(natHoleRespMsg.CandidateAddrs) == 0 {
+		return nil, fmt.Errorf("natHoleRespMsg get empty candidate addresses")
+	}
+	return natHoleRespMsg, nil
 }
 
-func (nc *Controller) CloseClient(name string) {
-	nc.mu.Lock()
-	defer nc.mu.Unlock()
-	delete(nc.clientCfgs, name)
+// MakeHole is used to make a NAT hole between client and visitor.
+func MakeHole(ctx context.Context, listenConn *net.UDPConn, m *msg.NatHoleResp, key []byte) (*net.UDPConn, *net.UDPAddr, error) {
+	xl := xlog.FromContextSafe(ctx)
+	transactionID := NewTransactionID()
+	sendToRangePortsFunc := func(conn *net.UDPConn, addr string) error {
+		return sendSidMessage(ctx, conn, m.Sid, transactionID, addr, key, m.DetectBehavior.TTL)
+	}
+
+	listenConns := []*net.UDPConn{listenConn}
+	var detectAddrs []string
+	if m.DetectBehavior.Role == DetectRoleSender {
+		// sender
+		if m.DetectBehavior.SendDelayMs > 0 {
+			time.Sleep(time.Duration(m.DetectBehavior.SendDelayMs) * time.Millisecond)
+		}
+		detectAddrs = m.AssistedAddrs
+		detectAddrs = append(detectAddrs, m.CandidateAddrs...)
+	} else {
+		// receiver
+		if len(m.DetectBehavior.CandidatePorts) == 0 {
+			detectAddrs = m.CandidateAddrs
+		}
+
+		if m.DetectBehavior.ListenRandomPorts > 0 {
+			for i := 0; i < m.DetectBehavior.ListenRandomPorts; i++ {
+				tmpConn, err := net.ListenUDP("udp4", nil)
+				if err != nil {
+					xl.Warn("listen random udp addr error: %v", err)
+					continue
+				}
+				listenConns = append(listenConns, tmpConn)
+			}
+		}
+	}
+
+	detectAddrs = lo.Uniq(detectAddrs)
+	for _, detectAddr := range detectAddrs {
+		for _, conn := range listenConns {
+			if err := sendSidMessage(ctx, conn, m.Sid, transactionID, detectAddr, key, m.DetectBehavior.TTL); err != nil {
+				xl.Trace("send sid message from %s to %s error: %v", conn.LocalAddr(), detectAddr, err)
+			}
+		}
+	}
+	if len(m.DetectBehavior.CandidatePorts) > 0 {
+		for _, conn := range listenConns {
+			sendSidMessageToRangePorts(ctx, conn, m.CandidateAddrs, m.DetectBehavior.CandidatePorts, sendToRangePortsFunc)
+		}
+	}
+	if m.DetectBehavior.SendRandomPorts > 0 {
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+		for i := range listenConns {
+			go sendSidMessageToRandomPorts(ctx, listenConns[i], m.CandidateAddrs, m.DetectBehavior.SendRandomPorts, sendToRangePortsFunc)
+		}
+	}
+
+	timeout := 5 * time.Second
+	if m.DetectBehavior.ReadTimeoutMs > 0 {
+		timeout = time.Duration(m.DetectBehavior.ReadTimeoutMs) * time.Millisecond
+	}
+
+	if len(listenConns) == 1 {
+		raddr, err := waitDetectMessage(ctx, listenConns[0], m.Sid, key, timeout, m.DetectBehavior.Role)
+		if err != nil {
+			return nil, nil, fmt.Errorf("wait detect message error: %v", err)
+		}
+		return listenConns[0], raddr, nil
+	}
+
+	type result struct {
+		lConn *net.UDPConn
+		raddr *net.UDPAddr
+	}
+	resultCh := make(chan result)
+	for _, conn := range listenConns {
+		go func(lConn *net.UDPConn) {
+			addr, err := waitDetectMessage(ctx, lConn, m.Sid, key, timeout, m.DetectBehavior.Role)
+			if err != nil {
+				lConn.Close()
+				return
+			}
+			select {
+			case resultCh <- result{lConn: lConn, raddr: addr}:
+			default:
+				lConn.Close()
+			}
+		}(conn)
+	}
+
+	select {
+	case result := <-resultCh:
+		return result.lConn, result.raddr, nil
+	case <-time.After(timeout):
+		return nil, nil, fmt.Errorf("wait detect message timeout")
+	case <-ctx.Done():
+		return nil, nil, fmt.Errorf("wait detect message canceled")
+	}
 }
 
-func (nc *Controller) Run() {
+func waitDetectMessage(
+	ctx context.Context, conn *net.UDPConn, sid string, key []byte,
+	timeout time.Duration, role string,
+) (*net.UDPAddr, error) {
+	xl := xlog.FromContextSafe(ctx)
 	for {
 		buf := pool.GetBuf(1024)
-		n, raddr, err := nc.listener.ReadFromUDP(buf)
+		_ = conn.SetReadDeadline(time.Now().Add(timeout))
+		n, raddr, err := conn.ReadFromUDP(buf)
+		_ = conn.SetReadDeadline(time.Time{})
 		if err != nil {
-			log.Trace("nat hole listener read from udp error: %v", err)
-			return
+			return nil, err
 		}
-
-		rd := bytes.NewReader(buf[:n])
-		rawMsg, err := msg.ReadMsg(rd)
-		if err != nil {
-			log.Trace("read nat hole message error: %v", err)
-			continue
-		}
-
-		switch m := rawMsg.(type) {
-		case *msg.NatHoleVisitor:
-			go nc.HandleVisitor(m, raddr)
-		case *msg.NatHoleClient:
-			go nc.HandleClient(m, raddr)
-		default:
-			log.Trace("error nat hole message type")
+		xl.Debug("get udp message local %s, from %s", conn.LocalAddr(), raddr)
+		var m msg.NatHoleSid
+		if err := DecodeMessageInto(buf[:n], key, &m); err != nil {
+			xl.Warn("decode sid message error: %v", err)
 			continue
 		}
 		pool.PutBuf(buf)
-	}
-}
 
-func (nc *Controller) GenSid() string {
-	t := time.Now().Unix()
-	id, _ := util.RandID()
-	return fmt.Sprintf("%d%s", t, id)
-}
-
-func (nc *Controller) HandleVisitor(m *msg.NatHoleVisitor, raddr *net.UDPAddr) {
-	sid := nc.GenSid()
-	session := &Session{
-		Sid:         sid,
-		VisitorAddr: raddr,
-		NotifyCh:    make(chan struct{}),
-	}
-	nc.mu.Lock()
-	clientCfg, ok := nc.clientCfgs[m.ProxyName]
-	if !ok {
-		nc.mu.Unlock()
-		errInfo := fmt.Sprintf("xtcp server for [%s] doesn't exist", m.ProxyName)
-		log.Debug(errInfo)
-		_, _ = nc.listener.WriteToUDP(nc.GenNatHoleResponse(nil, errInfo), raddr)
-		return
-	}
-	if m.SignKey != util.GetAuthKey(clientCfg.Sk, m.Timestamp) {
-		nc.mu.Unlock()
-		errInfo := fmt.Sprintf("xtcp connection of [%s] auth failed", m.ProxyName)
-		log.Debug(errInfo)
-		_, _ = nc.listener.WriteToUDP(nc.GenNatHoleResponse(nil, errInfo), raddr)
-		return
-	}
-
-	nc.sessions[sid] = session
-	nc.mu.Unlock()
-	log.Trace("handle visitor message, sid [%s]", sid)
-
-	defer func() {
-		nc.mu.Lock()
-		delete(nc.sessions, sid)
-		nc.mu.Unlock()
-	}()
-
-	err := errors.PanicToError(func() {
-		clientCfg.SidCh <- &SidRequest{
-			Sid:      sid,
-			NotifyCh: session.NotifyCh,
+		if m.Sid != sid {
+			xl.Warn("get sid message with wrong sid: %s, expect: %s", m.Sid, sid)
+			continue
 		}
-	})
+
+		if !m.Response {
+			// only wait for response messages if we are a sender
+			if role == DetectRoleSender {
+				continue
+			}
+
+			m.Response = true
+			buf2, err := EncodeMessage(&m, key)
+			if err != nil {
+				xl.Warn("encode sid message error: %v", err)
+				continue
+			}
+			_, _ = conn.WriteToUDP(buf2, raddr)
+		}
+		return raddr, nil
+	}
+}
+
+func sendSidMessage(
+	ctx context.Context, conn *net.UDPConn,
+	sid string, transactionID string, addr string, key []byte, ttl int,
+) error {
+	xl := xlog.FromContextSafe(ctx)
+	ttlStr := ""
+	if ttl > 0 {
+		ttlStr = fmt.Sprintf(" with ttl %d", ttl)
+	}
+	xl.Trace("send sid message from %s to %s%s", conn.LocalAddr(), addr, ttlStr)
+	raddr, err := net.ResolveUDPAddr("udp4", addr)
 	if err != nil {
-		return
+		return err
 	}
-
-	// Wait client connections.
-	select {
-	case <-session.NotifyCh:
-		resp := nc.GenNatHoleResponse(session, "")
-		log.Trace("send nat hole response to visitor")
-		_, _ = nc.listener.WriteToUDP(resp, raddr)
-	case <-time.After(time.Duration(NatHoleTimeout) * time.Second):
-		return
+	if transactionID == "" {
+		transactionID = NewTransactionID()
 	}
-}
-
-func (nc *Controller) HandleClient(m *msg.NatHoleClient, raddr *net.UDPAddr) {
-	nc.mu.RLock()
-	session, ok := nc.sessions[m.Sid]
-	nc.mu.RUnlock()
-	if !ok {
-		return
+	m := &msg.NatHoleSid{
+		TransactionID: transactionID,
+		Sid:           sid,
+		Response:      false,
+		Nonce:         strings.Repeat("0", rand.Intn(20)),
 	}
-	log.Trace("handle client message, sid [%s]", session.Sid)
-	session.ClientAddr = raddr
-
-	resp := nc.GenNatHoleResponse(session, "")
-	log.Trace("send nat hole response to client")
-	_, _ = nc.listener.WriteToUDP(resp, raddr)
-}
-
-func (nc *Controller) GenNatHoleResponse(session *Session, errInfo string) []byte {
-	var (
-		sid         string
-		visitorAddr string
-		clientAddr  string
-	)
-	if session != nil {
-		sid = session.Sid
-		visitorAddr = session.VisitorAddr.String()
-		clientAddr = session.ClientAddr.String()
-	}
-	m := &msg.NatHoleResp{
-		Sid:         sid,
-		VisitorAddr: visitorAddr,
-		ClientAddr:  clientAddr,
-		Error:       errInfo,
-	}
-	b := bytes.NewBuffer(nil)
-	err := msg.WriteMsg(b, m)
+	buf, err := EncodeMessage(m, key)
 	if err != nil {
-		return []byte("")
+		return err
 	}
-	return b.Bytes()
+	if ttl > 0 {
+		uConn := ipv4.NewConn(conn)
+		original, err := uConn.TTL()
+		if err != nil {
+			xl.Trace("get ttl error %v", err)
+			return err
+		}
+		xl.Trace("original ttl %d", original)
+
+		err = uConn.SetTTL(ttl)
+		if err != nil {
+			xl.Trace("set ttl error %v", err)
+		} else {
+			defer func() {
+				_ = uConn.SetTTL(original)
+			}()
+		}
+	}
+
+	if _, err := conn.WriteToUDP(buf, raddr); err != nil {
+		return err
+	}
+	return nil
 }
 
-type Session struct {
-	Sid         string
-	VisitorAddr *net.UDPAddr
-	ClientAddr  *net.UDPAddr
-
-	NotifyCh chan struct{}
+func sendSidMessageToRangePorts(
+	ctx context.Context, conn *net.UDPConn, addrs []string, ports []msg.PortsRange,
+	sendFunc func(*net.UDPConn, string) error,
+) {
+	xl := xlog.FromContextSafe(ctx)
+	for _, ip := range lo.Uniq(parseIPs(addrs)) {
+		for _, portsRange := range ports {
+			for i := portsRange.From; i <= portsRange.To; i++ {
+				detectAddr := net.JoinHostPort(ip, strconv.Itoa(i))
+				if err := sendFunc(conn, detectAddr); err != nil {
+					xl.Trace("send sid message from %s to %s error: %v", conn.LocalAddr(), detectAddr, err)
+				}
+				time.Sleep(5 * time.Millisecond)
+			}
+		}
+	}
 }
 
-type ClientCfg struct {
-	Name  string
-	Sk    string
-	SidCh chan *SidRequest
+func sendSidMessageToRandomPorts(
+	ctx context.Context, conn *net.UDPConn, addrs []string, count int,
+	sendFunc func(*net.UDPConn, string) error,
+) {
+	xl := xlog.FromContextSafe(ctx)
+	used := sets.New[int]()
+	getUnusedPort := func() int {
+		for i := 0; i < 10; i++ {
+			port := rand.Intn(65535-1024) + 1024
+			if !used.Has(port) {
+				used.Insert(port)
+				return port
+			}
+		}
+		return 0
+	}
+
+	for i := 0; i < count; i++ {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		port := getUnusedPort()
+		if port == 0 {
+			continue
+		}
+
+		for _, ip := range lo.Uniq(parseIPs(addrs)) {
+			detectAddr := net.JoinHostPort(ip, strconv.Itoa(port))
+			if err := sendFunc(conn, detectAddr); err != nil {
+				xl.Trace("send sid message from %s to %s error: %v", conn.LocalAddr(), detectAddr, err)
+			}
+			time.Sleep(time.Millisecond * 15)
+		}
+	}
+}
+
+func parseIPs(addrs []string) []string {
+	var ips []string
+	for _, addr := range addrs {
+		if ip, _, err := net.SplitHostPort(addr); err == nil {
+			ips = append(ips, ip)
+		}
+	}
+	return ips
 }

pkg/nathole/utils.go

@@ -0,0 +1,112 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nathole
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"strconv"
+
+	"github.com/fatedier/golib/crypto"
+	"github.com/pion/stun"
+
+	"github.com/fatedier/frp/pkg/msg"
+)
+
+func EncodeMessage(m msg.Message, key []byte) ([]byte, error) {
+	buffer := bytes.NewBuffer(nil)
+	if err := msg.WriteMsg(buffer, m); err != nil {
+		return nil, err
+	}
+
+	buf, err := crypto.Encode(buffer.Bytes(), key)
+	if err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+func DecodeMessageInto(data, key []byte, m msg.Message) error {
+	buf, err := crypto.Decode(data, key)
+	if err != nil {
+		return err
+	}
+
+	if err := msg.ReadMsgInto(bytes.NewReader(buf), m); err != nil {
+		return err
+	}
+	return nil
+}
+
+type ChangedAddress struct {
+	IP   net.IP
+	Port int
+}
+
+func (s *ChangedAddress) GetFrom(m *stun.Message) error {
+	a := (*stun.MappedAddress)(s)
+	return a.GetFromAs(m, stun.AttrChangedAddress)
+}
+
+func (s *ChangedAddress) String() string {
+	return net.JoinHostPort(s.IP.String(), strconv.Itoa(s.Port))
+}
+
+func ListAllLocalIPs() ([]net.IP, error) {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return nil, err
+	}
+	ips := make([]net.IP, 0, len(addrs))
+	for _, addr := range addrs {
+		ip, _, err := net.ParseCIDR(addr.String())
+		if err != nil {
+			continue
+		}
+		ips = append(ips, ip)
+	}
+	return ips, nil
+}
+
+func ListLocalIPsForNatHole(max int) ([]string, error) {
+	if max <= 0 {
+		return nil, fmt.Errorf("max must be greater than 0")
+	}
+
+	ips, err := ListAllLocalIPs()
+	if err != nil {
+		return nil, err
+	}
+
+	filtered := make([]string, 0, max)
+	for _, ip := range ips {
+		if len(filtered) >= max {
+			break
+		}
+
+		// ignore ipv6 address
+		if ip.To4() == nil {
+			continue
+		}
+		// ignore localhost IP
+		if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+			continue
+		}
+
+		filtered = append(filtered, ip.String())
+	}
+	return filtered, nil
+}

pkg/plugin/client/http_proxy.go

@@ -21,11 +21,13 @@ import (
 	"net"
 	"net/http"
 	"strings"
+	"time"
 
 	frpIo "github.com/fatedier/golib/io"
 	gnet "github.com/fatedier/golib/net"
 
 	frpNet "github.com/fatedier/frp/pkg/util/net"
+	"github.com/fatedier/frp/pkg/util/util"
 )
 
 const PluginHTTPProxy = "http_proxy"
@@ -179,7 +181,9 @@ func (hp *HTTPProxy) Auth(req *http.Request) bool {
 		return false
 	}
 
-	if pair[0] != hp.AuthUser || pair[1] != hp.AuthPasswd {
+	if !util.ConstantTimeEqString(pair[0], hp.AuthUser) ||
+		!util.ConstantTimeEqString(pair[1], hp.AuthPasswd) {
+		time.Sleep(200 * time.Millisecond)
 		return false
 	}
 	return true

pkg/plugin/client/static_file.go

@@ -18,6 +18,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"time"
 
 	"github.com/gorilla/mux"
 
@@ -64,7 +65,7 @@ func NewStaticFilePlugin(params map[string]string) (Plugin, error) {
 	}
 
 	router := mux.NewRouter()
-	router.Use(frpNet.NewHTTPAuthMiddleware(httpUser, httpPasswd).Middleware)
+	router.Use(frpNet.NewHTTPAuthMiddleware(httpUser, httpPasswd).SetAuthFailDelay(200 * time.Millisecond).Middleware)
 	router.PathPrefix(prefix).Handler(frpNet.MakeHTTPGzipHandler(http.StripPrefix(prefix, http.FileServer(http.Dir(localPath))))).Methods("GET")
 	sp.s = &http.Server{
 		Handler: router,

pkg/transport/message.go

@@ -0,0 +1,119 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transport
+
+import (
+	"context"
+	"reflect"
+	"sync"
+
+	"github.com/fatedier/golib/errors"
+
+	"github.com/fatedier/frp/pkg/msg"
+)
+
+type MessageTransporter interface {
+	Send(msg.Message) error
+	// Recv(ctx context.Context, laneKey string, msgType string) (Message, error)
+	// Do will first send msg, then recv msg with the same laneKey and specified msgType.
+	Do(ctx context.Context, req msg.Message, laneKey, recvMsgType string) (msg.Message, error)
+	Dispatch(m msg.Message, laneKey string) bool
+	DispatchWithType(m msg.Message, msgType, laneKey string) bool
+}
+
+func NewMessageTransporter(sendCh chan msg.Message) MessageTransporter {
+	return &transporterImpl{
+		sendCh:   sendCh,
+		registry: make(map[string]map[string]chan msg.Message),
+	}
+}
+
+type transporterImpl struct {
+	sendCh chan msg.Message
+
+	// First key is message type and second key is lane key.
+	// Dispatch will dispatch message to releated channel by its message type
+	// and lane key.
+	registry map[string]map[string]chan msg.Message
+	mu       sync.RWMutex
+}
+
+func (impl *transporterImpl) Send(m msg.Message) error {
+	return errors.PanicToError(func() {
+		impl.sendCh <- m
+	})
+}
+
+func (impl *transporterImpl) Do(ctx context.Context, req msg.Message, laneKey, recvMsgType string) (msg.Message, error) {
+	ch := make(chan msg.Message, 1)
+	defer close(ch)
+	unregisterFn := impl.registerMsgChan(ch, laneKey, recvMsgType)
+	defer unregisterFn()
+
+	if err := impl.Send(req); err != nil {
+		return nil, err
+	}
+
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case resp := <-ch:
+		return resp, nil
+	}
+}
+
+func (impl *transporterImpl) DispatchWithType(m msg.Message, msgType, laneKey string) bool {
+	var ch chan msg.Message
+	impl.mu.RLock()
+	byLaneKey, ok := impl.registry[msgType]
+	if ok {
+		ch = byLaneKey[laneKey]
+	}
+	impl.mu.RUnlock()
+
+	if ch == nil {
+		return false
+	}
+
+	if err := errors.PanicToError(func() {
+		ch <- m
+	}); err != nil {
+		return false
+	}
+	return true
+}
+
+func (impl *transporterImpl) Dispatch(m msg.Message, laneKey string) bool {
+	msgType := reflect.TypeOf(m).Elem().Name()
+	return impl.DispatchWithType(m, msgType, laneKey)
+}
+
+func (impl *transporterImpl) registerMsgChan(recvCh chan msg.Message, laneKey string, msgType string) (unregister func()) {
+	impl.mu.Lock()
+	byLaneKey, ok := impl.registry[msgType]
+	if !ok {
+		byLaneKey = make(map[string]chan msg.Message)
+		impl.registry[msgType] = byLaneKey
+	}
+	byLaneKey[laneKey] = recvCh
+	impl.mu.Unlock()
+
+	unregister = func() {
+		impl.mu.Lock()
+		delete(byLaneKey, laneKey)
+		impl.mu.Unlock()
+	}
+	return
+}

pkg/transport/tls.go

@@ -1,3 +1,17 @@
+// Copyright 2023 The frp Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package transport
 
 import (

pkg/util/net/conn.go

@@ -22,7 +22,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	quic "github.com/lucas-clemente/quic-go"
+	quic "github.com/quic-go/quic-go"
 
 	"github.com/fatedier/frp/pkg/util/xlog"
 )

pkg/util/net/http.go

@@ -19,6 +19,9 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"time"
+
+	"github.com/fatedier/frp/pkg/util/util"
 )
 
 type HTTPAuthWraper struct {
@@ -46,8 +49,9 @@ func (aw *HTTPAuthWraper) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 type HTTPAuthMiddleware struct {
-	user   string
-	passwd string
+	user          string
+	passwd        string
+	authFailDelay time.Duration
 }
 
 func NewHTTPAuthMiddleware(user, passwd string) *HTTPAuthMiddleware {
@@ -57,32 +61,28 @@ func NewHTTPAuthMiddleware(user, passwd string) *HTTPAuthMiddleware {
 	}
 }
 
+func (authMid *HTTPAuthMiddleware) SetAuthFailDelay(delay time.Duration) *HTTPAuthMiddleware {
+	authMid.authFailDelay = delay
+	return authMid
+}
+
 func (authMid *HTTPAuthMiddleware) Middleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		reqUser, reqPasswd, hasAuth := r.BasicAuth()
 		if (authMid.user == "" && authMid.passwd == "") ||
-			(hasAuth && reqUser == authMid.user && reqPasswd == authMid.passwd) {
+			(hasAuth && util.ConstantTimeEqString(reqUser, authMid.user) &&
+				util.ConstantTimeEqString(reqPasswd, authMid.passwd)) {
 			next.ServeHTTP(w, r)
 		} else {
+			if authMid.authFailDelay > 0 {
+				time.Sleep(authMid.authFailDelay)
+			}
 			w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
 			http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
 		}
 	})
 }
 
-func HTTPBasicAuth(h http.HandlerFunc, user, passwd string) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		reqUser, reqPasswd, hasAuth := r.BasicAuth()
-		if (user == "" && passwd == "") ||
-			(hasAuth && reqUser == user && reqPasswd == passwd) {
-			h.ServeHTTP(w, r)
-		} else {
-			w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`)
-			http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
-		}
-	}
-}
-
 type HTTPGzipWraper struct {
 	h http.Handler
 }

pkg/util/net/udp.go

@@ -256,3 +256,11 @@ func (l *UDPListener) Close() error {
 func (l *UDPListener) Addr() net.Addr {
 	return l.addr
 }
+
+// ConnectedUDPConn is a wrapper for net.UDPConn which converts WriteTo syscalls
+// to Write syscalls that are 4 times faster on some OS'es. This should only be
+// used for connections that were produced by a net.Dial* call.
+type ConnectedUDPConn struct{ *net.UDPConn }
+
+// WriteTo redirects all writes to the Write syscall, which is 4 times faster.
+func (c *ConnectedUDPConn) WriteTo(b []byte, addr net.Addr) (int, error) { return c.Write(b) }

pkg/util/tcpmux/httpconnect.go

@@ -31,18 +31,21 @@ import (
 type HTTPConnectTCPMuxer struct {
 	*vhost.Muxer
 
-	passthrough  bool
-	authRequired bool // Not supported until we really need this.
+	// If passthrough is set to true, the CONNECT request will be forwarded to the backend service.
+	// Otherwise, it will return an OK response to the client and forward the remaining content to the backend service.
+	passthrough bool
 }
 
 func NewHTTPConnectTCPMuxer(listener net.Listener, passthrough bool, timeout time.Duration) (*HTTPConnectTCPMuxer, error) {
-	ret := &HTTPConnectTCPMuxer{passthrough: passthrough, authRequired: false}
-	mux, err := vhost.NewMuxer(listener, ret.getHostFromHTTPConnect, nil, ret.sendConnectResponse, nil, timeout)
+	ret := &HTTPConnectTCPMuxer{passthrough: passthrough}
+	mux, err := vhost.NewMuxer(listener, ret.getHostFromHTTPConnect, timeout)
+	mux.SetCheckAuthFunc(ret.auth).
+		SetSuccessHookFunc(ret.sendConnectResponse)
 	ret.Muxer = mux
 	return ret, err
 }
 
-func (muxer *HTTPConnectTCPMuxer) readHTTPConnectRequest(rd io.Reader) (host string, httpUser string, err error) {
+func (muxer *HTTPConnectTCPMuxer) readHTTPConnectRequest(rd io.Reader) (host, httpUser, httpPwd string, err error) {
 	bufioReader := bufio.NewReader(rd)
 
 	req, err := http.ReadRequest(bufioReader)
@@ -58,7 +61,7 @@ func (muxer *HTTPConnectTCPMuxer) readHTTPConnectRequest(rd io.Reader) (host str
 	host, _ = util.CanonicalHost(req.Host)
 	proxyAuth := req.Header.Get("Proxy-Authorization")
 	if proxyAuth != "" {
-		httpUser, _, _ = util.ParseBasicAuth(proxyAuth)
+		httpUser, httpPwd, _ = util.ParseBasicAuth(proxyAuth)
 	}
 	return
 }
@@ -74,11 +77,26 @@ func (muxer *HTTPConnectTCPMuxer) sendConnectResponse(c net.Conn, reqInfo map[st
 	return res.Write(c)
 }
 
+func (muxer *HTTPConnectTCPMuxer) auth(c net.Conn, username, password string, reqInfo map[string]string) (bool, error) {
+	reqUsername := reqInfo["HTTPUser"]
+	reqPassword := reqInfo["HTTPPwd"]
+	if username == reqUsername && password == reqPassword {
+		return true, nil
+	}
+
+	resp := util.ProxyUnauthorizedResponse()
+	if resp.Body != nil {
+		defer resp.Body.Close()
+	}
+	_ = resp.Write(c)
+	return false, nil
+}
+
 func (muxer *HTTPConnectTCPMuxer) getHostFromHTTPConnect(c net.Conn) (net.Conn, map[string]string, error) {
 	reqInfoMap := make(map[string]string, 0)
 	sc, rd := gnet.NewSharedConn(c)
 
-	host, httpUser, err := muxer.readHTTPConnectRequest(rd)
+	host, httpUser, httpPwd, err := muxer.readHTTPConnectRequest(rd)
 	if err != nil {
 		return nil, reqInfoMap, err
 	}
@@ -86,18 +104,11 @@ func (muxer *HTTPConnectTCPMuxer) getHostFromHTTPConnect(c net.Conn) (net.Conn,
 	reqInfoMap["Host"] = host
 	reqInfoMap["Scheme"] = "tcp"
 	reqInfoMap["HTTPUser"] = httpUser
+	reqInfoMap["HTTPPwd"] = httpPwd
 
 	outConn := c
 	if muxer.passthrough {
 		outConn = sc
-		if muxer.authRequired && httpUser == "" {
-			resp := util.ProxyUnauthorizedResponse()
-			if resp.Body != nil {
-				defer resp.Body.Close()
-			}
-			_ = resp.Write(c)
-			outConn = c
-		}
 	}
 	return outConn, reqInfoMap, nil
 }

pkg/util/util/util.go

@@ -17,6 +17,7 @@ package util
 import (
 	"crypto/md5"
 	"crypto/rand"
+	"crypto/subtle"
 	"encoding/hex"
 	"fmt"
 	mathrand "math/rand"
@@ -28,25 +29,38 @@ import (
 
 // RandID return a rand string used in frp.
 func RandID() (id string, err error) {
-	return RandIDWithLen(8)
+	return RandIDWithLen(16)
 }
 
 // RandIDWithLen return a rand string with idLen length.
 func RandIDWithLen(idLen int) (id string, err error) {
-	b := make([]byte, idLen)
+	if idLen <= 0 {
+		return "", nil
+	}
+	b := make([]byte, idLen/2+1)
 	_, err = rand.Read(b)
 	if err != nil {
 		return
 	}
 
 	id = fmt.Sprintf("%x", b)
-	return
+	return id[:idLen], nil
+}
+
+// RandIDWithRandLen return a rand string with length between [start, end).
+func RandIDWithRandLen(start, end int) (id string, err error) {
+	if start >= end {
+		err = fmt.Errorf("start should be less than end")
+		return
+	}
+	idLen := mathrand.Intn(end-start) + start
+	return RandIDWithLen(idLen)
 }
 
 func GetAuthKey(token string, timestamp int64) (key string) {
-	token += fmt.Sprintf("%d", timestamp)
 	md5Ctx := md5.New()
 	md5Ctx.Write([]byte(token))
+	md5Ctx.Write([]byte(strconv.FormatInt(timestamp, 10)))
 	data := md5Ctx.Sum(nil)
 	return hex.EncodeToString(data)
 }
@@ -126,3 +140,7 @@ func RandomSleep(duration time.Duration, minRatio, maxRatio float64) time.Durati
 	time.Sleep(d)
 	return d
 }
+
+func ConstantTimeEqString(a, b string) bool {
+	return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
+}

pkg/util/util/util_test.go

@@ -14,10 +14,51 @@ func TestRandId(t *testing.T) {
 	assert.Equal(16, len(id))
 }
 
+func TestRandIDWithRandLen(t *testing.T) {
+	tests := []struct {
+		name      string
+		start     int
+		end       int
+		expectErr bool
+	}{
+		{
+			name:      "start and end are equal",
+			start:     5,
+			end:       5,
+			expectErr: true,
+		},
+		{
+			name:      "start is less than end",
+			start:     5,
+			end:       10,
+			expectErr: false,
+		},
+		{
+			name:      "start is greater than end",
+			start:     10,
+			end:       5,
+			expectErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert := assert.New(t)
+			id, err := RandIDWithRandLen(tt.start, tt.end)
+			if tt.expectErr {
+				assert.Error(err)
+			} else {
+				assert.NoError(err)
+				assert.GreaterOrEqual(len(id), tt.start)
+				assert.Less(len(id), tt.end)
+			}
+		})
+	}
+}
+
 func TestGetAuthKey(t *testing.T) {
 	assert := assert.New(t)
 	key := GetAuthKey("1234", 1488720000)
-	t.Log(key)
 	assert.Equal("6df41a43725f0c770fd56379e12acf8c", key)
 }
 

pkg/util/version/version.go

@@ -19,7 +19,7 @@ import (
 	"strings"
 )
 
-var version = "0.46.1"
+var version = "0.49.0"
 
 func Full() string {
 	return version

pkg/util/vhost/https.go

@@ -28,7 +28,10 @@ type HTTPSMuxer struct {
 }
 
 func NewHTTPSMuxer(listener net.Listener, timeout time.Duration) (*HTTPSMuxer, error) {
-	mux, err := NewMuxer(listener, GetHTTPSHostname, nil, nil, nil, timeout)
+	mux, err := NewMuxer(listener, GetHTTPSHostname, timeout)
+	if err != nil {
+		return nil, err
+	}
 	return &HTTPSMuxer{mux}, err
 }
 

pkg/util/vhost/resource.go

@@ -85,17 +85,3 @@ func notFoundResponse() *http.Response {
 	}
 	return res
 }
-
-func noAuthResponse() *http.Response {
-	header := make(map[string][]string)
-	header["WWW-Authenticate"] = []string{`Basic realm="Restricted"`}
-	res := &http.Response{
-		Status:     "401 Not authorized",
-		StatusCode: 401,
-		Proto:      "HTTP/1.1",
-		ProtoMajor: 1,
-		ProtoMinor: 1,
-		Header:     header,
-	}
-	return res
-}

pkg/util/vhost/vhost.go

@@ -43,43 +43,55 @@ type RequestRouteInfo struct {
 
 type (
 	muxFunc         func(net.Conn) (net.Conn, map[string]string, error)
-	httpAuthFunc    func(net.Conn, string, string, string) (bool, error)
+	authFunc        func(conn net.Conn, username, password string, reqInfoMap map[string]string) (bool, error)
 	hostRewriteFunc func(net.Conn, string) (net.Conn, error)
-	successFunc     func(net.Conn, map[string]string) error
+	successHookFunc func(net.Conn, map[string]string) error
 )
 
-// Muxer is only used for https and tcpmux proxy.
+// Muxer is a functional component used for https and tcpmux proxies.
+// It accepts connections and extracts vhost information from the beginning of the connection data.
+// It then routes the connection to its appropriate listener.
 type Muxer struct {
-	listener       net.Listener
-	timeout        time.Duration
+	listener net.Listener
+	timeout  time.Duration
+
 	vhostFunc      muxFunc
-	authFunc       httpAuthFunc
-	successFunc    successFunc
-	rewriteFunc    hostRewriteFunc
+	checkAuth      authFunc
+	successHook    successHookFunc
+	rewriteHost    hostRewriteFunc
 	registryRouter *Routers
 }
 
 func NewMuxer(
 	listener net.Listener,
 	vhostFunc muxFunc,
-	authFunc httpAuthFunc,
-	successFunc successFunc,
-	rewriteFunc hostRewriteFunc,
 	timeout time.Duration,
 ) (mux *Muxer, err error) {
 	mux = &Muxer{
 		listener:       listener,
 		timeout:        timeout,
 		vhostFunc:      vhostFunc,
-		authFunc:       authFunc,
-		successFunc:    successFunc,
-		rewriteFunc:    rewriteFunc,
 		registryRouter: NewRouters(),
 	}
 	go mux.run()
 	return mux, nil
 }
 
+func (v *Muxer) SetCheckAuthFunc(f authFunc) *Muxer {
+	v.checkAuth = f
+	return v
+}
+
+func (v *Muxer) SetSuccessHookFunc(f successHookFunc) *Muxer {
+	v.successHook = f
+	return v
+}
+
+func (v *Muxer) SetRewriteHostFunc(f hostRewriteFunc) *Muxer {
+	v.rewriteHost = f
+	return v
+}
+
 type ChooseEndpointFunc func() (string, error)
 
 type CreateConnFunc func(remoteAddr string) (net.Conn, error)
@@ -101,7 +113,7 @@ type RouteConfig struct {
 	CreateConnByEndpointFn CreateConnByEndpointFunc
 }
 
-// listen for a new domain name, if rewriteHost is not empty  and rewriteFunc is not nil
+// listen for a new domain name, if rewriteHost is not empty and rewriteHost func is not nil,
 // then rewrite the host header to rewriteHost
 func (v *Muxer) Listen(ctx context.Context, cfg *RouteConfig) (l *Listener, err error) {
 	l = &Listener{
@@ -109,8 +121,8 @@ func (v *Muxer) Listen(ctx context.Context, cfg *RouteConfig) (l *Listener, err
 		location:        cfg.Location,
 		routeByHTTPUser: cfg.RouteByHTTPUser,
 		rewriteHost:     cfg.RewriteHost,
-		userName:        cfg.Username,
-		passWord:        cfg.Password,
+		username:        cfg.Username,
+		password:        cfg.Password,
 		mux:             v,
 		accept:          make(chan net.Conn),
 		ctx:             ctx,
@@ -205,25 +217,20 @@ func (v *Muxer) handle(c net.Conn) {
 	}
 
 	xl := xlog.FromContextSafe(l.ctx)
-	if v.successFunc != nil {
-		if err := v.successFunc(c, reqInfoMap); err != nil {
+	if v.successHook != nil {
+		if err := v.successHook(c, reqInfoMap); err != nil {
 			xl.Info("success func failure on vhost connection: %v", err)
 			_ = c.Close()
 			return
 		}
 	}
 
-	// if authFunc is exist and username/password is set
+	// if checkAuth func is exist and username/password is set
 	// then verify user access
-	if l.mux.authFunc != nil && l.userName != "" && l.passWord != "" {
-		bAccess, err := l.mux.authFunc(c, l.userName, l.passWord, reqInfoMap["Authorization"])
-		if !bAccess || err != nil {
-			xl.Debug("check http Authorization failed")
-			res := noAuthResponse()
-			if res.Body != nil {
-				defer res.Body.Close()
-			}
-			_ = res.Write(c)
+	if l.mux.checkAuth != nil && l.username != "" {
+		ok, err := l.mux.checkAuth(c, l.username, l.password, reqInfoMap)
+		if !ok || err != nil {
+			xl.Debug("auth failed for user: %s", l.username)
 			_ = c.Close()
 			return
 		}
@@ -249,8 +256,8 @@ type Listener struct {
 	location        string
 	routeByHTTPUser string
 	rewriteHost     string
-	userName        string
-	passWord        string
+	username        string
+	password        string
 	mux             *Muxer // for closing Muxer
 	accept          chan net.Conn
 	ctx             context.Context
@@ -263,11 +270,11 @@ func (l *Listener) Accept() (net.Conn, error) {
 		return nil, fmt.Errorf("Listener closed")
 	}
 
-	// if rewriteFunc is exist
+	// if rewriteHost func is exist
 	// rewrite http requests with a modified host header
 	// if l.rewriteHost is empty, nothing to do
-	if l.mux.rewriteFunc != nil {
-		sConn, err := l.mux.rewriteFunc(conn, l.rewriteHost)
+	if l.mux.rewriteHost != nil {
+		sConn, err := l.mux.rewriteHost(conn, l.rewriteHost)
 		if err != nil {
 			xl.Warn("host header rewrite failed: %v", err)
 			return nil, fmt.Errorf("host header rewrite failed")

server/control.go

@@ -33,6 +33,7 @@ import (
 	frpErr "github.com/fatedier/frp/pkg/errors"
 	"github.com/fatedier/frp/pkg/msg"
 	plugin "github.com/fatedier/frp/pkg/plugin/server"
+	"github.com/fatedier/frp/pkg/transport"
 	"github.com/fatedier/frp/pkg/util/util"
 	"github.com/fatedier/frp/pkg/util/version"
 	"github.com/fatedier/frp/pkg/util/xlog"
@@ -82,6 +83,16 @@ func (cm *ControlManager) GetByID(runID string) (ctl *Control, ok bool) {
 	return
 }
 
+func (cm *ControlManager) Close() error {
+	cm.mu.Lock()
+	defer cm.mu.Unlock()
+	for _, ctl := range cm.ctlsByRunID {
+		ctl.Close()
+	}
+	cm.ctlsByRunID = make(map[string]*Control)
+	return nil
+}
+
 type Control struct {
 	// all resource managers and controllers
 	rc *controller.ResourceController
@@ -95,6 +106,9 @@ type Control struct {
 	// verifies authentication based on selected method
 	authVerifier auth.Verifier
 
+	// other components can use this to communicate with client
+	msgTransporter transport.MessageTransporter
+
 	// login message
 	loginMsg *msg.Login
 
@@ -158,7 +172,7 @@ func NewControl(
 	if poolCount > int(serverCfg.MaxPoolCount) {
 		poolCount = int(serverCfg.MaxPoolCount)
 	}
-	return &Control{
+	ctl := &Control{
 		rc:              rc,
 		pxyManager:      pxyManager,
 		pluginManager:   pluginManager,
@@ -182,15 +196,16 @@ func NewControl(
 		xl:              xlog.FromContextSafe(ctx),
 		ctx:             ctx,
 	}
+	ctl.msgTransporter = transport.NewMessageTransporter(ctl.sendCh)
+	return ctl
 }
 
 // Start send a login success message to client and start working.
 func (ctl *Control) Start() {
 	loginRespMsg := &msg.LoginResp{
-		Version:       version.Full(),
-		RunID:         ctl.runID,
-		ServerUDPPort: ctl.serverCfg.BindUDPPort,
-		Error:         "",
+		Version: version.Full(),
+		RunID:   ctl.runID,
+		Error:   "",
 	}
 	_ = msg.WriteMsg(ctl.conn, loginRespMsg)
 
@@ -204,6 +219,18 @@ func (ctl *Control) Start() {
 	go ctl.stoper()
 }
 
+func (ctl *Control) Close() error {
+	ctl.allShutdown.Start()
+	return nil
+}
+
+func (ctl *Control) Replaced(newCtl *Control) {
+	xl := ctl.xl
+	xl.Info("Replaced by client [%s]", newCtl.runID)
+	ctl.runID = ""
+	ctl.allShutdown.Start()
+}
+
 func (ctl *Control) RegisterWorkConn(conn net.Conn) error {
 	xl := ctl.xl
 	defer func() {
@@ -275,13 +302,6 @@ func (ctl *Control) GetWorkConn() (workConn net.Conn, err error) {
 	return
 }
 
-func (ctl *Control) Replaced(newCtl *Control) {
-	xl := ctl.xl
-	xl.Info("Replaced by client [%s]", newCtl.runID)
-	ctl.runID = ""
-	ctl.allShutdown.Start()
-}
-
 func (ctl *Control) writer() {
 	xl := ctl.xl
 	defer func() {
@@ -465,6 +485,12 @@ func (ctl *Control) manager() {
 					metrics.Server.NewProxy(m.ProxyName, m.ProxyType)
 				}
 				ctl.sendCh <- resp
+			case *msg.NatHoleVisitor:
+				go ctl.HandleNatHoleVisitor(m)
+			case *msg.NatHoleClient:
+				go ctl.HandleNatHoleClient(m)
+			case *msg.NatHoleReport:
+				go ctl.HandleNatHoleReport(m)
 			case *msg.CloseProxy:
 				_ = ctl.CloseProxy(m)
 				xl.Info("close proxy [%s] success", m.ProxyName)
@@ -497,6 +523,18 @@ func (ctl *Control) manager() {
 	}
 }
 
+func (ctl *Control) HandleNatHoleVisitor(m *msg.NatHoleVisitor) {
+	ctl.rc.NatHoleController.HandleVisitor(m, ctl.msgTransporter)
+}
+
+func (ctl *Control) HandleNatHoleClient(m *msg.NatHoleClient) {
+	ctl.rc.NatHoleController.HandleClient(m, ctl.msgTransporter)
+}
+
+func (ctl *Control) HandleNatHoleReport(m *msg.NatHoleReport) {
+	ctl.rc.NatHoleController.HandleReport(m)
+}
+
 func (ctl *Control) RegisterProxy(pxyMsg *msg.NewProxy) (remoteAddr string, err error) {
 	var pxyConf config.ProxyConf
 	// Load configures from NewProxy message and check.
@@ -514,7 +552,7 @@ func (ctl *Control) RegisterProxy(pxyMsg *msg.NewProxy) (remoteAddr string, err
 
 	// NewProxy will return a interface Proxy.
 	// In fact it create different proxies by different proxy type, we just call run() here.
-	pxy, err := proxy.NewProxy(ctl.ctx, userInfo, ctl.rc, ctl.poolCount, ctl.GetWorkConn, pxyConf, ctl.serverCfg)
+	pxy, err := proxy.NewProxy(ctl.ctx, userInfo, ctl.rc, ctl.poolCount, ctl.GetWorkConn, pxyConf, ctl.serverCfg, ctl.loginMsg)
 	if err != nil {
 		return remoteAddr, err
 	}

server/dashboard.go

@@ -50,7 +50,7 @@ func (svr *Service) RunDashboardServer(address string) (err error) {
 	subRouter := router.NewRoute().Subrouter()
 
 	user, passwd := svr.cfg.DashboardUser, svr.cfg.DashboardPwd
-	subRouter.Use(frpNet.NewHTTPAuthMiddleware(user, passwd).Middleware)
+	subRouter.Use(frpNet.NewHTTPAuthMiddleware(user, passwd).SetAuthFailDelay(200 * time.Millisecond).Middleware)
 
 	// metrics
 	if svr.cfg.EnablePrometheus {

server/dashboard_api.go

@@ -33,16 +33,19 @@ type GeneralResponse struct {
 }
 
 type serverInfoResp struct {
-	Version           string `json:"version"`
-	BindPort          int    `json:"bind_port"`
-	BindUDPPort       int    `json:"bind_udp_port"`
-	VhostHTTPPort     int    `json:"vhost_http_port"`
-	VhostHTTPSPort    int    `json:"vhost_https_port"`
-	KCPBindPort       int    `json:"kcp_bind_port"`
-	SubdomainHost     string `json:"subdomain_host"`
-	MaxPoolCount      int64  `json:"max_pool_count"`
-	MaxPortsPerClient int64  `json:"max_ports_per_client"`
-	HeartBeatTimeout  int64  `json:"heart_beat_timeout"`
+	Version               string `json:"version"`
+	BindPort              int    `json:"bind_port"`
+	VhostHTTPPort         int    `json:"vhost_http_port"`
+	VhostHTTPSPort        int    `json:"vhost_https_port"`
+	TCPMuxHTTPConnectPort int    `json:"tcpmux_httpconnect_port"`
+	KCPBindPort           int    `json:"kcp_bind_port"`
+	QUICBindPort          int    `json:"quic_bind_port"`
+	SubdomainHost         string `json:"subdomain_host"`
+	MaxPoolCount          int64  `json:"max_pool_count"`
+	MaxPortsPerClient     int64  `json:"max_ports_per_client"`
+	HeartBeatTimeout      int64  `json:"heart_beat_timeout"`
+	AllowPortsStr         string `json:"allow_ports_str,omitempty"`
+	TLSOnly               bool   `json:"tls_only,omitempty"`
 
 	TotalTrafficIn  int64            `json:"total_traffic_in"`
 	TotalTrafficOut int64            `json:"total_traffic_out"`
@@ -70,16 +73,19 @@ func (svr *Service) APIServerInfo(w http.ResponseWriter, r *http.Request) {
 	log.Info("Http request: [%s]", r.URL.Path)
 	serverStats := mem.StatsCollector.GetServer()
 	svrResp := serverInfoResp{
-		Version:           version.Full(),
-		BindPort:          svr.cfg.BindPort,
-		BindUDPPort:       svr.cfg.BindUDPPort,
-		VhostHTTPPort:     svr.cfg.VhostHTTPPort,
-		VhostHTTPSPort:    svr.cfg.VhostHTTPSPort,
-		KCPBindPort:       svr.cfg.KCPBindPort,
-		SubdomainHost:     svr.cfg.SubDomainHost,
-		MaxPoolCount:      svr.cfg.MaxPoolCount,
-		MaxPortsPerClient: svr.cfg.MaxPortsPerClient,
-		HeartBeatTimeout:  svr.cfg.HeartbeatTimeout,
+		Version:               version.Full(),
+		BindPort:              svr.cfg.BindPort,
+		VhostHTTPPort:         svr.cfg.VhostHTTPPort,
+		VhostHTTPSPort:        svr.cfg.VhostHTTPSPort,
+		TCPMuxHTTPConnectPort: svr.cfg.TCPMuxHTTPConnectPort,
+		KCPBindPort:           svr.cfg.KCPBindPort,
+		QUICBindPort:          svr.cfg.QUICBindPort,
+		SubdomainHost:         svr.cfg.SubDomainHost,
+		MaxPoolCount:          svr.cfg.MaxPoolCount,
+		MaxPortsPerClient:     svr.cfg.MaxPortsPerClient,
+		HeartBeatTimeout:      svr.cfg.HeartbeatTimeout,
+		AllowPortsStr:         svr.cfg.AllowPortsStr,
+		TLSOnly:               svr.cfg.TLSOnly,
 
 		TotalTrafficIn:  serverStats.TotalTrafficIn,
 		TotalTrafficOut: serverStats.TotalTrafficOut,
@@ -157,6 +163,7 @@ func getConfByType(proxyType string) interface{} {
 type ProxyStatsInfo struct {
 	Name            string      `json:"name"`
 	Conf            interface{} `json:"conf"`
+	ClientVersion   string      `json:"client_version,omitempty"`
 	TodayTrafficIn  int64       `json:"today_traffic_in"`
 	TodayTrafficOut int64       `json:"today_traffic_out"`
 	CurConns        int64       `json:"cur_conns"`
@@ -208,6 +215,9 @@ func (svr *Service) getProxyStatsByType(proxyType string) (proxyInfos []*ProxySt
 				continue
 			}
 			proxyInfo.Status = consts.Online
+			if pxy.GetLoginMsg() != nil {
+				proxyInfo.ClientVersion = pxy.GetLoginMsg().Version
+			}
 		} else {
 			proxyInfo.Status = consts.Offline
 		}

server/group/tcpmux.go

@@ -81,6 +81,8 @@ type TCPMuxGroup struct {
 	groupKey        string
 	domain          string
 	routeByHTTPUser string
+	username        string
+	password        string
 
 	acceptCh chan net.Conn
 	tcpMuxLn net.Listener
@@ -120,6 +122,8 @@ func (tmg *TCPMuxGroup) HTTPConnectListen(
 		tmg.groupKey = groupKey
 		tmg.domain = routeConfig.Domain
 		tmg.routeByHTTPUser = routeConfig.RouteByHTTPUser
+		tmg.username = routeConfig.Username
+		tmg.password = routeConfig.Password
 		tmg.tcpMuxLn = tcpMuxLn
 		tmg.lns = append(tmg.lns, ln)
 		if tmg.acceptCh == nil {
@@ -128,7 +132,10 @@ func (tmg *TCPMuxGroup) HTTPConnectListen(
 		go tmg.worker()
 	} else {
 		// route config in the same group must be equal
-		if tmg.group != group || tmg.domain != routeConfig.Domain || tmg.routeByHTTPUser != routeConfig.RouteByHTTPUser {
+		if tmg.group != group || tmg.domain != routeConfig.Domain ||
+			tmg.routeByHTTPUser != routeConfig.RouteByHTTPUser ||
+			tmg.username != routeConfig.Username ||
+			tmg.password != routeConfig.Password {
 			return nil, ErrGroupParamsInvalid
 		}
 		if tmg.groupKey != groupKey {

server/proxy/http.go

@@ -20,8 +20,10 @@ import (
 	"strings"
 
 	frpIo "github.com/fatedier/golib/io"
+	"golang.org/x/time/rate"
 
 	"github.com/fatedier/frp/pkg/config"
+	"github.com/fatedier/frp/pkg/util/limit"
 	frpNet "github.com/fatedier/frp/pkg/util/net"
 	"github.com/fatedier/frp/pkg/util/util"
 	"github.com/fatedier/frp/pkg/util/vhost"
@@ -135,6 +137,10 @@ func (pxy *HTTPProxy) GetConf() config.ProxyConf {
 	return pxy.cfg
 }
 
+func (pxy *HTTPProxy) GetLimiter() *rate.Limiter {
+	return pxy.limiter
+}
+
 func (pxy *HTTPProxy) GetRealConn(remoteAddr string) (workConn net.Conn, err error) {
 	xl := pxy.xl
 	rAddr, errRet := net.ResolveTCPAddr("tcp", remoteAddr)
@@ -160,6 +166,13 @@ func (pxy *HTTPProxy) GetRealConn(remoteAddr string) (workConn net.Conn, err err
 	if pxy.cfg.UseCompression {
 		rwc = frpIo.WithCompression(rwc)
 	}
+
+	if pxy.GetLimiter() != nil {
+		rwc = frpIo.WrapReadWriteCloser(limit.NewReader(rwc, pxy.GetLimiter()), limit.NewWriter(rwc, pxy.GetLimiter()), func() error {
+			return rwc.Close()
+		})
+	}
+
 	workConn = frpNet.WrapReadWriteCloserToConn(rwc, tmpConn)
 	workConn = frpNet.WrapStatsConn(workConn, pxy.updateStatsAfterClosedConn)
 	metrics.Server.OpenConnection(pxy.GetName(), pxy.GetConf().GetBaseInfo().ProxyType)