new e2e framework (#1835)

test/e2e/e2e.go

@@ -0,0 +1,61 @@
+package e2e
+
+import (
+	"testing"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/utils/log"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+	"github.com/onsi/gomega"
+)
+
+var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
+	setupSuite()
+	return nil
+}, func(data []byte) {
+	// Run on all Ginkgo nodes
+	setupSuitePerGinkgoNode()
+})
+
+var _ = ginkgo.SynchronizedAfterSuite(func() {
+	CleanupSuite()
+}, func() {
+	AfterSuiteActions()
+})
+
+// RunE2ETests checks configuration parameters (specified through flags) and then runs
+// E2E tests using the Ginkgo runner.
+// If a "report directory" is specified, one or more JUnit test reports will be
+// generated in this directory, and cluster logs will also be saved.
+// This function is called on each Ginkgo node in parallel mode.
+func RunE2ETests(t *testing.T) {
+	gomega.RegisterFailHandler(framework.Fail)
+
+	log.Info("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)
+	ginkgo.RunSpecs(t, "frp e2e suite")
+}
+
+// setupSuite is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
+// There are certain operations we only want to run once per overall test invocation
+// (such as deleting old namespaces, or verifying that all system pods are running.
+// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
+// to ensure that these operations only run on the first parallel Ginkgo node.
+//
+// This function takes two parameters: one function which runs on only the first Ginkgo node,
+// returning an opaque byte array, and then a second function which runs on all Ginkgo nodes,
+// accepting the byte array.
+func setupSuite() {
+	// Run only on Ginkgo node 1
+	// TODO
+}
+
+// setupSuitePerGinkgoNode is the boilerplate that can be used to setup ginkgo test suites, on the SynchronizedBeforeSuite step.
+// There are certain operations we only want to run once per overall test invocation on each Ginkgo node
+// such as making some global variables accessible to all parallel executions
+// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
+// Ref: https://onsi.github.io/ginkgo/#parallel-specs
+func setupSuitePerGinkgoNode() {
+	// config.GinkgoConfig.ParallelNode
+}

test/e2e/e2e_test.go

@@ -0,0 +1,34 @@
+package e2e
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/utils/log"
+)
+
+// handleFlags sets up all flags and parses the command line.
+func handleFlags() {
+	framework.RegisterCommonFlags(flag.CommandLine)
+	flag.Parse()
+}
+
+func TestMain(m *testing.M) {
+	// Register test flags, then parse flags.
+	handleFlags()
+
+	if err := framework.ValidateTestContext(&framework.TestContext); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	log.InitLog("console", "", framework.TestContext.LogLevel, 0, true)
+	os.Exit(m.Run())
+}
+
+func TestE2E(t *testing.T) {
+	RunE2ETests(t)
+}

test/e2e/examples.go

@@ -0,0 +1,40 @@
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/framework"
+	"github.com/fatedier/frp/test/e2e/framework/consts"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var connTimeout = 5 * time.Second
+
+var _ = Describe("[Feature: Example]", func() {
+	f := framework.NewDefaultFramework()
+
+	Describe("TCP", func() {
+		It("Expose a TCP echo server", func() {
+			serverConf := `
+			[common]
+			bind_port = {{ .PortServer }}
+			`
+
+			clientConf := fmt.Sprintf(`
+			[common]
+			server_port = {{ .PortServer }}
+
+			[tcp]
+			type = tcp
+			local_port = {{ .%s }}
+			remote_port = {{ .PortTCP }}
+			`, framework.TCPEchoServerPort)
+
+			f.RunProcesses([]string{serverConf}, []string{clientConf})
+
+			framework.ExpectTCPReuqest(f.UsedPorts["PortTCP"], []byte(consts.TestString), []byte(consts.TestString), connTimeout)
+		})
+	})
+})

test/e2e/framework/cleanup.go

@@ -0,0 +1,59 @@
+package framework
+
+import (
+	"sync"
+)
+
+// CleanupActionHandle is an integer pointer type for handling cleanup action
+type CleanupActionHandle *int
+type cleanupFuncHandle struct {
+	actionHandle CleanupActionHandle
+	actionHook   func()
+}
+
+var cleanupActionsLock sync.Mutex
+var cleanupHookList = []cleanupFuncHandle{}
+
+// AddCleanupAction installs a function that will be called in the event of the
+// whole test being terminated.  This allows arbitrary pieces of the overall
+// test to hook into SynchronizedAfterSuite().
+// The hooks are called in last-in-first-out order.
+func AddCleanupAction(fn func()) CleanupActionHandle {
+	p := CleanupActionHandle(new(int))
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	c := cleanupFuncHandle{actionHandle: p, actionHook: fn}
+	cleanupHookList = append([]cleanupFuncHandle{c}, cleanupHookList...)
+	return p
+}
+
+// RemoveCleanupAction removes a function that was installed by
+// AddCleanupAction.
+func RemoveCleanupAction(p CleanupActionHandle) {
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	for i, item := range cleanupHookList {
+		if item.actionHandle == p {
+			cleanupHookList = append(cleanupHookList[:i], cleanupHookList[i+1:]...)
+			break
+		}
+	}
+}
+
+// RunCleanupActions runs all functions installed by AddCleanupAction.  It does
+// not remove them (see RemoveCleanupAction) but it does run unlocked, so they
+// may remove themselves.
+func RunCleanupActions() {
+	list := []func(){}
+	func() {
+		cleanupActionsLock.Lock()
+		defer cleanupActionsLock.Unlock()
+		for _, p := range cleanupHookList {
+			list = append(list, p.actionHook)
+		}
+	}()
+	// Run unlocked.
+	for _, fn := range list {
+		fn()
+	}
+}

test/e2e/framework/client.go

@@ -0,0 +1,11 @@
+package framework
+
+type FRPClient struct {
+	port int
+}
+
+func (f *Framework) FRPClient(port int) *FRPClient {
+	return &FRPClient{
+		port: port,
+	}
+}

test/e2e/framework/consts/consts.go

@@ -0,0 +1,5 @@
+package consts
+
+const (
+	TestString = "frp is a fast reverse proxy to help you expose a local server behind a NAT or firewall to the internet."
+)

test/e2e/framework/expect.go

@@ -0,0 +1,55 @@
+package framework
+
+import (
+	"github.com/onsi/gomega"
+)
+
+// ExpectEqual expects the specified two are the same, otherwise an exception raises
+func ExpectEqual(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.Equal(extra), explain...)
+}
+
+// ExpectEqualValues expects the specified two are the same, it not strict about type
+func ExpectEqualValues(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.BeEquivalentTo(extra), explain...)
+}
+
+// ExpectNotEqual expects the specified two are not the same, otherwise an exception raises
+func ExpectNotEqual(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).NotTo(gomega.Equal(extra), explain...)
+}
+
+// ExpectError expects an error happens, otherwise an exception raises
+func ExpectError(err error, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, err).To(gomega.HaveOccurred(), explain...)
+}
+
+// ExpectNoError checks if "err" is set, and if so, fails assertion while logging the error.
+func ExpectNoError(err error, explain ...interface{}) {
+	ExpectNoErrorWithOffset(1, err, explain...)
+}
+
+// ExpectNoErrorWithOffset checks if "err" is set, and if so, fails assertion while logging the error at "offset" levels above its caller
+// (for example, for call chain f -> g -> ExpectNoErrorWithOffset(1, ...) error would be logged for "f").
+func ExpectNoErrorWithOffset(offset int, err error, explain ...interface{}) {
+	gomega.ExpectWithOffset(1+offset, err).NotTo(gomega.HaveOccurred(), explain...)
+}
+
+// ExpectConsistOf expects actual contains precisely the extra elements.  The ordering of the elements does not matter.
+func ExpectConsistOf(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.ConsistOf(extra), explain...)
+}
+
+// ExpectHaveKey expects the actual map has the key in the keyset
+func ExpectHaveKey(actual interface{}, key interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.HaveKey(key), explain...)
+}
+
+// ExpectEmpty expects actual is empty
+func ExpectEmpty(actual interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.BeEmpty(), explain...)
+}
+
+func ExpectTrue(actual interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).Should(gomega.BeTrue(), explain...)
+}

test/e2e/framework/framework.go

@@ -0,0 +1,167 @@
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
+	"text/template"
+
+	"github.com/fatedier/frp/test/e2e/pkg/port"
+	"github.com/fatedier/frp/test/e2e/pkg/process"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+)
+
+type Options struct {
+	TotalParallelNode int
+	CurrentNodeIndex  int
+	FromPortIndex     int
+	ToPortIndex       int
+}
+
+type Framework struct {
+	TempDirectory string
+	UsedPorts     map[string]int
+
+	portAllocator *port.Allocator
+
+	// Multiple mock servers used for e2e testing.
+	mockServers *MockServers
+
+	// To make sure that this framework cleans up after itself, no matter what,
+	// we install a Cleanup action before each test and clear it after.  If we
+	// should abort, the AfterSuite hook should run all Cleanup actions.
+	cleanupHandle CleanupActionHandle
+
+	// beforeEachStarted indicates that BeforeEach has started
+	beforeEachStarted bool
+
+	serverConfPaths []string
+	serverProcesses []*process.Process
+	clientConfPaths []string
+	clientProcesses []*process.Process
+}
+
+func NewDefaultFramework() *Framework {
+	options := Options{
+		TotalParallelNode: config.GinkgoConfig.ParallelTotal,
+		CurrentNodeIndex:  config.GinkgoConfig.ParallelNode,
+		FromPortIndex:     20000,
+		ToPortIndex:       50000,
+	}
+	return NewFramework(options)
+}
+
+func NewFramework(opt Options) *Framework {
+	f := &Framework{
+		portAllocator: port.NewAllocator(opt.FromPortIndex, opt.ToPortIndex, opt.TotalParallelNode, opt.CurrentNodeIndex-1),
+	}
+	f.mockServers = NewMockServers(f.portAllocator)
+	if err := f.mockServers.Run(); err != nil {
+		Failf("%v", err)
+	}
+
+	ginkgo.BeforeEach(f.BeforeEach)
+	ginkgo.AfterEach(f.AfterEach)
+	return f
+}
+
+// BeforeEach create a temp directory.
+func (f *Framework) BeforeEach() {
+	f.beforeEachStarted = true
+
+	f.cleanupHandle = AddCleanupAction(f.AfterEach)
+
+	dir, err := ioutil.TempDir(os.TempDir(), "frpe2e-test-*")
+	ExpectNoError(err)
+	f.TempDirectory = dir
+}
+
+func (f *Framework) AfterEach() {
+	if !f.beforeEachStarted {
+		return
+	}
+
+	RemoveCleanupAction(f.cleanupHandle)
+
+	os.RemoveAll(f.TempDirectory)
+	f.TempDirectory = ""
+	f.UsedPorts = nil
+	f.serverConfPaths = nil
+	f.clientConfPaths = nil
+	for _, p := range f.serverProcesses {
+		p.Stop()
+	}
+	for _, p := range f.clientProcesses {
+		p.Stop()
+	}
+	f.serverProcesses = nil
+	f.clientProcesses = nil
+}
+
+var portRegex = regexp.MustCompile(`{{ \.Port.*? }}`)
+
+// RenderPortsTemplate render templates with ports.
+//
+// Local: {{ .Port1 }}
+// Target: {{ .Port2 }}
+//
+// return rendered content and all allocated ports.
+func (f *Framework) genPortsFromTemplates(templates []string) (ports map[string]int, err error) {
+	ports = make(map[string]int)
+	for _, t := range templates {
+		arrs := portRegex.FindAllString(t, -1)
+		for _, str := range arrs {
+			str = strings.TrimPrefix(str, "{{ .")
+			str = strings.TrimSuffix(str, " }}")
+			str = strings.TrimSpace(str)
+			ports[str] = 0
+		}
+	}
+	defer func() {
+		if err != nil {
+			for _, port := range ports {
+				f.portAllocator.Release(port)
+			}
+		}
+	}()
+
+	for name := range ports {
+		port := f.portAllocator.Get()
+		if port <= 0 {
+			return nil, fmt.Errorf("can't allocate port")
+		}
+		ports[name] = port
+	}
+	return
+
+}
+
+func (f *Framework) RenderTemplates(templates []string) (outs []string, ports map[string]int, err error) {
+	ports, err = f.genPortsFromTemplates(templates)
+	if err != nil {
+		return
+	}
+
+	params := f.mockServers.GetTemplateParams()
+	for name, port := range ports {
+		params[name] = port
+	}
+
+	for _, t := range templates {
+		tmpl, err := template.New("").Parse(t)
+		if err != nil {
+			return nil, nil, err
+		}
+		buffer := bytes.NewBuffer(nil)
+		if err = tmpl.Execute(buffer, params); err != nil {
+			return nil, nil, err
+		}
+		outs = append(outs, buffer.String())
+	}
+	return
+}

test/e2e/framework/ginkgowrapper/wrapper.go

@@ -0,0 +1,80 @@
+// Package ginkgowrapper wraps Ginkgo Fail and Skip functions to panic
+// with structured data instead of a constant string.
+package ginkgowrapper
+
+import (
+	"bufio"
+	"bytes"
+	"regexp"
+	"runtime"
+	"runtime/debug"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+)
+
+// FailurePanic is the value that will be panicked from Fail.
+type FailurePanic struct {
+	Message        string // The failure message passed to Fail
+	Filename       string // The filename that is the source of the failure
+	Line           int    // The line number of the filename that is the source of the failure
+	FullStackTrace string // A full stack trace starting at the source of the failure
+}
+
+// String makes FailurePanic look like the old Ginkgo panic when printed.
+func (FailurePanic) String() string { return ginkgo.GINKGO_PANIC }
+
+// Fail wraps ginkgo.Fail so that it panics with more useful
+// information about the failure. This function will panic with a
+// FailurePanic.
+func Fail(message string, callerSkip ...int) {
+	skip := 1
+	if len(callerSkip) > 0 {
+		skip += callerSkip[0]
+	}
+
+	_, file, line, _ := runtime.Caller(skip)
+	fp := FailurePanic{
+		Message:        message,
+		Filename:       file,
+		Line:           line,
+		FullStackTrace: pruneStack(skip),
+	}
+
+	defer func() {
+		e := recover()
+		if e != nil {
+			panic(fp)
+		}
+	}()
+
+	ginkgo.Fail(message, skip)
+}
+
+// ginkgo adds a lot of test running infrastructure to the stack, so
+// we filter those out
+var stackSkipPattern = regexp.MustCompile(`onsi/ginkgo`)
+
+func pruneStack(skip int) string {
+	skip += 2 // one for pruneStack and one for debug.Stack
+	stack := debug.Stack()
+	scanner := bufio.NewScanner(bytes.NewBuffer(stack))
+	var prunedStack []string
+
+	// skip the top of the stack
+	for i := 0; i < 2*skip+1; i++ {
+		scanner.Scan()
+	}
+
+	for scanner.Scan() {
+		if stackSkipPattern.Match(scanner.Bytes()) {
+			scanner.Scan() // these come in pairs
+		} else {
+			prunedStack = append(prunedStack, scanner.Text())
+			scanner.Scan() // these come in pairs
+			prunedStack = append(prunedStack, scanner.Text())
+		}
+	}
+
+	return strings.Join(prunedStack, "\n")
+}

test/e2e/framework/log.go

@@ -0,0 +1,94 @@
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"runtime/debug"
+	"time"
+
+	"github.com/onsi/ginkgo"
+
+	e2eginkgowrapper "github.com/fatedier/frp/test/e2e/framework/ginkgowrapper"
+)
+
+func nowStamp() string {
+	return time.Now().Format(time.StampMilli)
+}
+
+func log(level string, format string, args ...interface{}) {
+	fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
+}
+
+// Logf logs the info.
+func Logf(format string, args ...interface{}) {
+	log("INFO", format, args...)
+}
+
+// Failf logs the fail info, including a stack trace.
+func Failf(format string, args ...interface{}) {
+	FailfWithOffset(1, format, args...)
+}
+
+// FailfWithOffset calls "Fail" and logs the error with a stack trace that starts at "offset" levels above its caller
+// (for example, for call chain f -> g -> FailfWithOffset(1, ...) error would be logged for "f").
+func FailfWithOffset(offset int, format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	skip := offset + 1
+	log("FAIL", "%s\n\nFull Stack Trace\n%s", msg, PrunedStack(skip))
+	e2eginkgowrapper.Fail(nowStamp()+": "+msg, skip)
+}
+
+// Fail is a replacement for ginkgo.Fail which logs the problem as it occurs
+// together with a stack trace and then calls ginkgowrapper.Fail.
+func Fail(msg string, callerSkip ...int) {
+	skip := 1
+	if len(callerSkip) > 0 {
+		skip += callerSkip[0]
+	}
+	log("FAIL", "%s\n\nFull Stack Trace\n%s", msg, PrunedStack(skip))
+	e2eginkgowrapper.Fail(nowStamp()+": "+msg, skip)
+}
+
+var codeFilterRE = regexp.MustCompile(`/github.com/onsi/ginkgo/`)
+
+// PrunedStack is a wrapper around debug.Stack() that removes information
+// about the current goroutine and optionally skips some of the initial stack entries.
+// With skip == 0, the returned stack will start with the caller of PruneStack.
+// From the remaining entries it automatically filters out useless ones like
+// entries coming from Ginkgo.
+//
+// This is a modified copy of PruneStack in https://github.com/onsi/ginkgo/blob/f90f37d87fa6b1dd9625e2b1e83c23ffae3de228/internal/codelocation/code_location.go#L25:
+// - simplified API and thus renamed (calls debug.Stack() instead of taking a parameter)
+// - source code filtering updated to be specific to Kubernetes
+// - optimized to use bytes and in-place slice filtering from
+//   https://github.com/golang/go/wiki/SliceTricks#filter-in-place
+func PrunedStack(skip int) []byte {
+	fullStackTrace := debug.Stack()
+	stack := bytes.Split(fullStackTrace, []byte("\n"))
+	// Ensure that the even entries are the method names and the
+	// the odd entries the source code information.
+	if len(stack) > 0 && bytes.HasPrefix(stack[0], []byte("goroutine ")) {
+		// Ignore "goroutine 29 [running]:" line.
+		stack = stack[1:]
+	}
+	// The "+2" is for skipping over:
+	// - runtime/debug.Stack()
+	// - PrunedStack()
+	skip += 2
+	if len(stack) > 2*skip {
+		stack = stack[2*skip:]
+	}
+	n := 0
+	for i := 0; i < len(stack)/2; i++ {
+		// We filter out based on the source code file name.
+		if !codeFilterRE.Match([]byte(stack[i*2+1])) {
+			stack[n] = stack[i*2]
+			stack[n+1] = stack[i*2+1]
+			n += 2
+		}
+	}
+	stack = stack[:n]
+
+	return bytes.Join(stack, []byte("\n"))
+}

test/e2e/framework/mockservers.go

@@ -0,0 +1,60 @@
+package framework
+
+import (
+	"github.com/fatedier/frp/test/e2e/mock/echoserver"
+	"github.com/fatedier/frp/test/e2e/pkg/port"
+)
+
+const (
+	TCPEchoServerPort = "TCPEchoServerPort"
+	UDPEchoServerPort = "UDPEchoServerPort"
+)
+
+type MockServers struct {
+	tcpEchoServer *echoserver.Server
+	udpEchoServer *echoserver.Server
+}
+
+func NewMockServers(portAllocator *port.Allocator) *MockServers {
+	s := &MockServers{}
+	tcpPort := portAllocator.Get()
+	udpPort := portAllocator.Get()
+	s.tcpEchoServer = echoserver.New(echoserver.Options{
+		Type:      echoserver.TCP,
+		BindAddr:  "127.0.0.1",
+		BindPort:  int32(tcpPort),
+		RepeatNum: 1,
+	})
+	s.udpEchoServer = echoserver.New(echoserver.Options{
+		Type:      echoserver.UDP,
+		BindAddr:  "127.0.0.1",
+		BindPort:  int32(udpPort),
+		RepeatNum: 1,
+	})
+	return s
+}
+
+func (m *MockServers) Run() error {
+	if err := m.tcpEchoServer.Run(); err != nil {
+		return err
+	}
+	if err := m.udpEchoServer.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *MockServers) GetTemplateParams() map[string]interface{} {
+	ret := make(map[string]interface{})
+	ret[TCPEchoServerPort] = m.tcpEchoServer.GetOptions().BindPort
+	ret[UDPEchoServerPort] = m.udpEchoServer.GetOptions().BindPort
+	return ret
+}
+
+func (m *MockServers) GetParam(key string) interface{} {
+	params := m.GetTemplateParams()
+	if v, ok := params[key]; ok {
+		return v
+	}
+	return nil
+}

test/e2e/framework/process.go

@@ -0,0 +1,59 @@
+package framework
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/pkg/process"
+	flog "github.com/fatedier/frp/utils/log"
+)
+
+func GenerateConfigFile(path string, content string) error {
+	return ioutil.WriteFile(path, []byte(content), 0666)
+}
+
+// RunProcesses run multiple processes from templates.
+// The first template should always be frps.
+func (f *Framework) RunProcesses(serverTemplates []string, clientTemplates []string) {
+	templates := make([]string, 0, len(serverTemplates)+len(clientTemplates))
+	for _, t := range serverTemplates {
+		templates = append(templates, t)
+	}
+	for _, t := range clientTemplates {
+		templates = append(templates, t)
+	}
+	outs, ports, err := f.RenderTemplates(templates)
+	ExpectNoError(err)
+	ExpectTrue(len(templates) > 0)
+
+	f.UsedPorts = ports
+
+	for i := range serverTemplates {
+		path := filepath.Join(f.TempDirectory, fmt.Sprintf("frp-e2e-server-%d", i))
+		err = ioutil.WriteFile(path, []byte(outs[i]), 0666)
+		ExpectNoError(err)
+		flog.Trace("[%s] %s", path, outs[i])
+		p := process.New(TestContext.FRPServerPath, []string{"-c", path})
+		f.serverConfPaths = append(f.serverConfPaths, path)
+		f.serverProcesses = append(f.serverProcesses, p)
+		err = p.Start()
+		ExpectNoError(err)
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	for i := range clientTemplates {
+		index := i + len(serverTemplates)
+		path := filepath.Join(f.TempDirectory, fmt.Sprintf("frp-e2e-client-%d", i))
+		err = ioutil.WriteFile(path, []byte(outs[index]), 0666)
+		ExpectNoError(err)
+		flog.Trace("[%s] %s", path, outs[index])
+		p := process.New(TestContext.FRPClientPath, []string{"-c", path})
+		f.clientConfPaths = append(f.clientConfPaths, path)
+		f.clientProcesses = append(f.clientProcesses, p)
+		err = p.Start()
+		ExpectNoError(err)
+		time.Sleep(500 * time.Millisecond)
+	}
+}

test/e2e/framework/request.go

@@ -0,0 +1,13 @@
+package framework
+
+import (
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/pkg/request"
+)
+
+func ExpectTCPReuqest(port int, in, out []byte, timeout time.Duration) {
+	res, err := request.SendTCPRequest(port, in, timeout)
+	ExpectNoError(err)
+	ExpectEqual(string(out), res)
+}

test/e2e/framework/test_context.go

@@ -0,0 +1,42 @@
+package framework
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+type TestContextType struct {
+	FRPClientPath string
+	FRPServerPath string
+	LogLevel      string
+}
+
+var TestContext TestContextType
+
+// RegisterCommonFlags registers flags common to all e2e test suites.
+// The flag set can be flag.CommandLine (if desired) or a custom
+// flag set that then gets passed to viperconfig.ViperizeFlags.
+//
+// The other Register*Flags methods below can be used to add more
+// test-specific flags. However, those settings then get added
+// regardless whether the test is actually in the test suite.
+//
+func RegisterCommonFlags(flags *flag.FlagSet) {
+	flags.StringVar(&TestContext.FRPClientPath, "frpc-path", "../../bin/frpc", "The frp client binary to use.")
+	flags.StringVar(&TestContext.FRPServerPath, "frps-path", "../../bin/frps", "The frp server binary to use.")
+	flags.StringVar(&TestContext.LogLevel, "log-level", "debug", "Log level.")
+}
+
+func ValidateTestContext(t *TestContextType) error {
+	if t.FRPClientPath == "" || t.FRPServerPath == "" {
+		return fmt.Errorf("frpc and frps binary path can't be empty")
+	}
+	if _, err := os.Stat(t.FRPClientPath); err != nil {
+		return fmt.Errorf("load frpc-path error: %v", err)
+	}
+	if _, err := os.Stat(t.FRPServerPath); err != nil {
+		return fmt.Errorf("load frps-path error: %v", err)
+	}
+	return nil
+}

test/e2e/framework/util.go

@@ -0,0 +1,14 @@
+package framework
+
+import (
+	"github.com/google/uuid"
+)
+
+// RunID is a unique identifier of the e2e run.
+// Beware that this ID is not the same for all tests in the e2e run, because each Ginkgo node creates it separately.
+var RunID string
+
+func init() {
+	uuid, _ := uuid.NewUUID()
+	RunID = uuid.String()
+}

test/e2e/mock/echoserver/echoserver.go

@@ -0,0 +1,111 @@
+package echoserver
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	fnet "github.com/fatedier/frp/utils/net"
+)
+
+type ServerType string
+
+const (
+	TCP  ServerType = "tcp"
+	UDP  ServerType = "udp"
+	Unix ServerType = "unix"
+)
+
+type Options struct {
+	Type              ServerType
+	BindAddr          string
+	BindPort          int32
+	RepeatNum         int
+	SpecifiedResponse string
+}
+
+type Server struct {
+	opt Options
+
+	l net.Listener
+}
+
+func New(opt Options) *Server {
+	if opt.Type == "" {
+		opt.Type = TCP
+	}
+	if opt.BindAddr == "" {
+		opt.BindAddr = "127.0.0.1"
+	}
+	if opt.RepeatNum <= 0 {
+		opt.RepeatNum = 1
+	}
+	return &Server{
+		opt: opt,
+	}
+}
+
+func (s *Server) GetOptions() Options {
+	return s.opt
+}
+
+func (s *Server) Run() error {
+	if err := s.initListener(); err != nil {
+		return err
+	}
+
+	go func() {
+		for {
+			c, err := s.l.Accept()
+			if err != nil {
+				return
+			}
+			go s.handle(c)
+		}
+	}()
+	return nil
+}
+
+func (s *Server) Close() error {
+	if s.l != nil {
+		return s.l.Close()
+	}
+	return nil
+}
+
+func (s *Server) initListener() (err error) {
+	switch s.opt.Type {
+	case TCP:
+		s.l, err = net.Listen("tcp", fmt.Sprintf("%s:%d", s.opt.BindAddr, s.opt.BindPort))
+	case UDP:
+		s.l, err = fnet.ListenUDP(s.opt.BindAddr, int(s.opt.BindPort))
+	case Unix:
+		s.l, err = net.Listen("unix", s.opt.BindAddr)
+	default:
+		return fmt.Errorf("unknown server type: %s", s.opt.Type)
+	}
+	if err != nil {
+		return
+	}
+	return nil
+}
+
+func (s *Server) handle(c net.Conn) {
+	defer c.Close()
+
+	buf := make([]byte, 2048)
+	for {
+		n, err := c.Read(buf)
+		if err != nil {
+			return
+		}
+
+		var response string
+		if len(s.opt.SpecifiedResponse) > 0 {
+			response = s.opt.SpecifiedResponse
+		} else {
+			response = strings.Repeat(string(buf[:n]), s.opt.RepeatNum)
+		}
+		c.Write([]byte(response))
+	}
+}

test/e2e/pkg/port/port.go

@@ -0,0 +1,57 @@
+package port
+
+import (
+	"fmt"
+	"net"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+type Allocator struct {
+	reserved sets.Int
+	used     sets.Int
+}
+
+// NewAllocator return a port allocator for testing.
+// Example: from: 10, to: 20, mod 4, index 1
+// Reserved ports: 13, 17
+func NewAllocator(from int, to int, mod int, index int) *Allocator {
+	pa := &Allocator{
+		reserved: sets.NewInt(),
+		used:     sets.NewInt(),
+	}
+	for i := from; i <= to; i++ {
+		if i%mod == index {
+			pa.reserved.Insert(i)
+		}
+	}
+	return pa
+}
+
+func (pa *Allocator) Get() int {
+	for i := 0; i < 10; i++ {
+		port, _ := pa.reserved.PopAny()
+		if port == 0 {
+			return 0
+		}
+
+		// TODO: Distinguish between TCP and UDP
+		l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+		if err != nil {
+			// Maybe not controlled by us, mark it used.
+			pa.used.Insert(port)
+			continue
+		}
+		l.Close()
+		pa.used.Insert(port)
+		return port
+	}
+	return 0
+}
+
+func (pa *Allocator) Release(port int) {
+	if pa.used.Has(port) {
+		pa.used.Delete(port)
+		pa.reserved.Insert(port)
+	}
+}

test/e2e/pkg/process/process.go

@@ -0,0 +1,47 @@
+package process
+
+import (
+	"bytes"
+	"context"
+	"os/exec"
+)
+
+type Process struct {
+	cmd         *exec.Cmd
+	cancel      context.CancelFunc
+	errorOutput *bytes.Buffer
+
+	beforeStopHandler func()
+}
+
+func New(path string, params []string) *Process {
+	ctx, cancel := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(ctx, path, params...)
+	p := &Process{
+		cmd:    cmd,
+		cancel: cancel,
+	}
+	p.errorOutput = bytes.NewBufferString("")
+	cmd.Stderr = p.errorOutput
+	return p
+}
+
+func (p *Process) Start() error {
+	return p.cmd.Start()
+}
+
+func (p *Process) Stop() error {
+	if p.beforeStopHandler != nil {
+		p.beforeStopHandler()
+	}
+	p.cancel()
+	return p.cmd.Wait()
+}
+
+func (p *Process) ErrorOutput() string {
+	return p.errorOutput.String()
+}
+
+func (p *Process) SetBeforeStopHandler(fn func()) {
+	p.beforeStopHandler = fn
+}

test/e2e/pkg/request/request.go

@@ -0,0 +1,31 @@
+package request
+
+import (
+	"fmt"
+	"net"
+	"time"
+)
+
+func SendTCPRequest(port int, content []byte, timeout time.Duration) (res string, err error) {
+	c, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+	if err != nil {
+		err = fmt.Errorf("connect to tcp server error: %v", err)
+		return
+	}
+	defer c.Close()
+
+	c.SetDeadline(time.Now().Add(timeout))
+	return sendTCPRequestByConn(c, content)
+}
+
+func sendTCPRequestByConn(c net.Conn, content []byte) (res string, err error) {
+	c.Write(content)
+
+	buf := make([]byte, 2048)
+	n, errRet := c.Read(buf)
+	if errRet != nil {
+		err = fmt.Errorf("read from tcp error: %v", errRet)
+		return
+	}
+	return string(buf[:n]), nil
+}

test/e2e/suites.go

@@ -0,0 +1,16 @@
+package e2e
+
+// CleanupSuite is the boilerplate that can be used after tests on ginkgo were run, on the SynchronizedAfterSuite step.
+// Similar to SynchronizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs).
+// Here, the order of functions is reversed; first, the function which runs everywhere,
+// and then the function that only runs on the first Ginkgo node.
+func CleanupSuite() {
+	// Run on all Ginkgo nodes
+	// TODO
+}
+
+// AfterSuiteActions are actions that are run on ginkgo's SynchronizedAfterSuite
+func AfterSuiteActions() {
+	// Run only Ginkgo on node 1
+	// TODO
+}