new e2e framework (#1835)

test/e2e/framework/cleanup.go

@@ -0,0 +1,59 @@
+package framework
+
+import (
+	"sync"
+)
+
+// CleanupActionHandle is an integer pointer type for handling cleanup action
+type CleanupActionHandle *int
+type cleanupFuncHandle struct {
+	actionHandle CleanupActionHandle
+	actionHook   func()
+}
+
+var cleanupActionsLock sync.Mutex
+var cleanupHookList = []cleanupFuncHandle{}
+
+// AddCleanupAction installs a function that will be called in the event of the
+// whole test being terminated.  This allows arbitrary pieces of the overall
+// test to hook into SynchronizedAfterSuite().
+// The hooks are called in last-in-first-out order.
+func AddCleanupAction(fn func()) CleanupActionHandle {
+	p := CleanupActionHandle(new(int))
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	c := cleanupFuncHandle{actionHandle: p, actionHook: fn}
+	cleanupHookList = append([]cleanupFuncHandle{c}, cleanupHookList...)
+	return p
+}
+
+// RemoveCleanupAction removes a function that was installed by
+// AddCleanupAction.
+func RemoveCleanupAction(p CleanupActionHandle) {
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	for i, item := range cleanupHookList {
+		if item.actionHandle == p {
+			cleanupHookList = append(cleanupHookList[:i], cleanupHookList[i+1:]...)
+			break
+		}
+	}
+}
+
+// RunCleanupActions runs all functions installed by AddCleanupAction.  It does
+// not remove them (see RemoveCleanupAction) but it does run unlocked, so they
+// may remove themselves.
+func RunCleanupActions() {
+	list := []func(){}
+	func() {
+		cleanupActionsLock.Lock()
+		defer cleanupActionsLock.Unlock()
+		for _, p := range cleanupHookList {
+			list = append(list, p.actionHook)
+		}
+	}()
+	// Run unlocked.
+	for _, fn := range list {
+		fn()
+	}
+}

test/e2e/framework/client.go

@@ -0,0 +1,11 @@
+package framework
+
+type FRPClient struct {
+	port int
+}
+
+func (f *Framework) FRPClient(port int) *FRPClient {
+	return &FRPClient{
+		port: port,
+	}
+}

test/e2e/framework/consts/consts.go

@@ -0,0 +1,5 @@
+package consts
+
+const (
+	TestString = "frp is a fast reverse proxy to help you expose a local server behind a NAT or firewall to the internet."
+)

test/e2e/framework/expect.go

@@ -0,0 +1,55 @@
+package framework
+
+import (
+	"github.com/onsi/gomega"
+)
+
+// ExpectEqual expects the specified two are the same, otherwise an exception raises
+func ExpectEqual(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.Equal(extra), explain...)
+}
+
+// ExpectEqualValues expects the specified two are the same, it not strict about type
+func ExpectEqualValues(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.BeEquivalentTo(extra), explain...)
+}
+
+// ExpectNotEqual expects the specified two are not the same, otherwise an exception raises
+func ExpectNotEqual(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).NotTo(gomega.Equal(extra), explain...)
+}
+
+// ExpectError expects an error happens, otherwise an exception raises
+func ExpectError(err error, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, err).To(gomega.HaveOccurred(), explain...)
+}
+
+// ExpectNoError checks if "err" is set, and if so, fails assertion while logging the error.
+func ExpectNoError(err error, explain ...interface{}) {
+	ExpectNoErrorWithOffset(1, err, explain...)
+}
+
+// ExpectNoErrorWithOffset checks if "err" is set, and if so, fails assertion while logging the error at "offset" levels above its caller
+// (for example, for call chain f -> g -> ExpectNoErrorWithOffset(1, ...) error would be logged for "f").
+func ExpectNoErrorWithOffset(offset int, err error, explain ...interface{}) {
+	gomega.ExpectWithOffset(1+offset, err).NotTo(gomega.HaveOccurred(), explain...)
+}
+
+// ExpectConsistOf expects actual contains precisely the extra elements.  The ordering of the elements does not matter.
+func ExpectConsistOf(actual interface{}, extra interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.ConsistOf(extra), explain...)
+}
+
+// ExpectHaveKey expects the actual map has the key in the keyset
+func ExpectHaveKey(actual interface{}, key interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.HaveKey(key), explain...)
+}
+
+// ExpectEmpty expects actual is empty
+func ExpectEmpty(actual interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).To(gomega.BeEmpty(), explain...)
+}
+
+func ExpectTrue(actual interface{}, explain ...interface{}) {
+	gomega.ExpectWithOffset(1, actual).Should(gomega.BeTrue(), explain...)
+}

test/e2e/framework/framework.go

@@ -0,0 +1,167 @@
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strings"
+	"text/template"
+
+	"github.com/fatedier/frp/test/e2e/pkg/port"
+	"github.com/fatedier/frp/test/e2e/pkg/process"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/config"
+)
+
+type Options struct {
+	TotalParallelNode int
+	CurrentNodeIndex  int
+	FromPortIndex     int
+	ToPortIndex       int
+}
+
+type Framework struct {
+	TempDirectory string
+	UsedPorts     map[string]int
+
+	portAllocator *port.Allocator
+
+	// Multiple mock servers used for e2e testing.
+	mockServers *MockServers
+
+	// To make sure that this framework cleans up after itself, no matter what,
+	// we install a Cleanup action before each test and clear it after.  If we
+	// should abort, the AfterSuite hook should run all Cleanup actions.
+	cleanupHandle CleanupActionHandle
+
+	// beforeEachStarted indicates that BeforeEach has started
+	beforeEachStarted bool
+
+	serverConfPaths []string
+	serverProcesses []*process.Process
+	clientConfPaths []string
+	clientProcesses []*process.Process
+}
+
+func NewDefaultFramework() *Framework {
+	options := Options{
+		TotalParallelNode: config.GinkgoConfig.ParallelTotal,
+		CurrentNodeIndex:  config.GinkgoConfig.ParallelNode,
+		FromPortIndex:     20000,
+		ToPortIndex:       50000,
+	}
+	return NewFramework(options)
+}
+
+func NewFramework(opt Options) *Framework {
+	f := &Framework{
+		portAllocator: port.NewAllocator(opt.FromPortIndex, opt.ToPortIndex, opt.TotalParallelNode, opt.CurrentNodeIndex-1),
+	}
+	f.mockServers = NewMockServers(f.portAllocator)
+	if err := f.mockServers.Run(); err != nil {
+		Failf("%v", err)
+	}
+
+	ginkgo.BeforeEach(f.BeforeEach)
+	ginkgo.AfterEach(f.AfterEach)
+	return f
+}
+
+// BeforeEach create a temp directory.
+func (f *Framework) BeforeEach() {
+	f.beforeEachStarted = true
+
+	f.cleanupHandle = AddCleanupAction(f.AfterEach)
+
+	dir, err := ioutil.TempDir(os.TempDir(), "frpe2e-test-*")
+	ExpectNoError(err)
+	f.TempDirectory = dir
+}
+
+func (f *Framework) AfterEach() {
+	if !f.beforeEachStarted {
+		return
+	}
+
+	RemoveCleanupAction(f.cleanupHandle)
+
+	os.RemoveAll(f.TempDirectory)
+	f.TempDirectory = ""
+	f.UsedPorts = nil
+	f.serverConfPaths = nil
+	f.clientConfPaths = nil
+	for _, p := range f.serverProcesses {
+		p.Stop()
+	}
+	for _, p := range f.clientProcesses {
+		p.Stop()
+	}
+	f.serverProcesses = nil
+	f.clientProcesses = nil
+}
+
+var portRegex = regexp.MustCompile(`{{ \.Port.*? }}`)
+
+// RenderPortsTemplate render templates with ports.
+//
+// Local: {{ .Port1 }}
+// Target: {{ .Port2 }}
+//
+// return rendered content and all allocated ports.
+func (f *Framework) genPortsFromTemplates(templates []string) (ports map[string]int, err error) {
+	ports = make(map[string]int)
+	for _, t := range templates {
+		arrs := portRegex.FindAllString(t, -1)
+		for _, str := range arrs {
+			str = strings.TrimPrefix(str, "{{ .")
+			str = strings.TrimSuffix(str, " }}")
+			str = strings.TrimSpace(str)
+			ports[str] = 0
+		}
+	}
+	defer func() {
+		if err != nil {
+			for _, port := range ports {
+				f.portAllocator.Release(port)
+			}
+		}
+	}()
+
+	for name := range ports {
+		port := f.portAllocator.Get()
+		if port <= 0 {
+			return nil, fmt.Errorf("can't allocate port")
+		}
+		ports[name] = port
+	}
+	return
+
+}
+
+func (f *Framework) RenderTemplates(templates []string) (outs []string, ports map[string]int, err error) {
+	ports, err = f.genPortsFromTemplates(templates)
+	if err != nil {
+		return
+	}
+
+	params := f.mockServers.GetTemplateParams()
+	for name, port := range ports {
+		params[name] = port
+	}
+
+	for _, t := range templates {
+		tmpl, err := template.New("").Parse(t)
+		if err != nil {
+			return nil, nil, err
+		}
+		buffer := bytes.NewBuffer(nil)
+		if err = tmpl.Execute(buffer, params); err != nil {
+			return nil, nil, err
+		}
+		outs = append(outs, buffer.String())
+	}
+	return
+}

test/e2e/framework/ginkgowrapper/wrapper.go

@@ -0,0 +1,80 @@
+// Package ginkgowrapper wraps Ginkgo Fail and Skip functions to panic
+// with structured data instead of a constant string.
+package ginkgowrapper
+
+import (
+	"bufio"
+	"bytes"
+	"regexp"
+	"runtime"
+	"runtime/debug"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+)
+
+// FailurePanic is the value that will be panicked from Fail.
+type FailurePanic struct {
+	Message        string // The failure message passed to Fail
+	Filename       string // The filename that is the source of the failure
+	Line           int    // The line number of the filename that is the source of the failure
+	FullStackTrace string // A full stack trace starting at the source of the failure
+}
+
+// String makes FailurePanic look like the old Ginkgo panic when printed.
+func (FailurePanic) String() string { return ginkgo.GINKGO_PANIC }
+
+// Fail wraps ginkgo.Fail so that it panics with more useful
+// information about the failure. This function will panic with a
+// FailurePanic.
+func Fail(message string, callerSkip ...int) {
+	skip := 1
+	if len(callerSkip) > 0 {
+		skip += callerSkip[0]
+	}
+
+	_, file, line, _ := runtime.Caller(skip)
+	fp := FailurePanic{
+		Message:        message,
+		Filename:       file,
+		Line:           line,
+		FullStackTrace: pruneStack(skip),
+	}
+
+	defer func() {
+		e := recover()
+		if e != nil {
+			panic(fp)
+		}
+	}()
+
+	ginkgo.Fail(message, skip)
+}
+
+// ginkgo adds a lot of test running infrastructure to the stack, so
+// we filter those out
+var stackSkipPattern = regexp.MustCompile(`onsi/ginkgo`)
+
+func pruneStack(skip int) string {
+	skip += 2 // one for pruneStack and one for debug.Stack
+	stack := debug.Stack()
+	scanner := bufio.NewScanner(bytes.NewBuffer(stack))
+	var prunedStack []string
+
+	// skip the top of the stack
+	for i := 0; i < 2*skip+1; i++ {
+		scanner.Scan()
+	}
+
+	for scanner.Scan() {
+		if stackSkipPattern.Match(scanner.Bytes()) {
+			scanner.Scan() // these come in pairs
+		} else {
+			prunedStack = append(prunedStack, scanner.Text())
+			scanner.Scan() // these come in pairs
+			prunedStack = append(prunedStack, scanner.Text())
+		}
+	}
+
+	return strings.Join(prunedStack, "\n")
+}

test/e2e/framework/log.go

@@ -0,0 +1,94 @@
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"runtime/debug"
+	"time"
+
+	"github.com/onsi/ginkgo"
+
+	e2eginkgowrapper "github.com/fatedier/frp/test/e2e/framework/ginkgowrapper"
+)
+
+func nowStamp() string {
+	return time.Now().Format(time.StampMilli)
+}
+
+func log(level string, format string, args ...interface{}) {
+	fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...)
+}
+
+// Logf logs the info.
+func Logf(format string, args ...interface{}) {
+	log("INFO", format, args...)
+}
+
+// Failf logs the fail info, including a stack trace.
+func Failf(format string, args ...interface{}) {
+	FailfWithOffset(1, format, args...)
+}
+
+// FailfWithOffset calls "Fail" and logs the error with a stack trace that starts at "offset" levels above its caller
+// (for example, for call chain f -> g -> FailfWithOffset(1, ...) error would be logged for "f").
+func FailfWithOffset(offset int, format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	skip := offset + 1
+	log("FAIL", "%s\n\nFull Stack Trace\n%s", msg, PrunedStack(skip))
+	e2eginkgowrapper.Fail(nowStamp()+": "+msg, skip)
+}
+
+// Fail is a replacement for ginkgo.Fail which logs the problem as it occurs
+// together with a stack trace and then calls ginkgowrapper.Fail.
+func Fail(msg string, callerSkip ...int) {
+	skip := 1
+	if len(callerSkip) > 0 {
+		skip += callerSkip[0]
+	}
+	log("FAIL", "%s\n\nFull Stack Trace\n%s", msg, PrunedStack(skip))
+	e2eginkgowrapper.Fail(nowStamp()+": "+msg, skip)
+}
+
+var codeFilterRE = regexp.MustCompile(`/github.com/onsi/ginkgo/`)
+
+// PrunedStack is a wrapper around debug.Stack() that removes information
+// about the current goroutine and optionally skips some of the initial stack entries.
+// With skip == 0, the returned stack will start with the caller of PruneStack.
+// From the remaining entries it automatically filters out useless ones like
+// entries coming from Ginkgo.
+//
+// This is a modified copy of PruneStack in https://github.com/onsi/ginkgo/blob/f90f37d87fa6b1dd9625e2b1e83c23ffae3de228/internal/codelocation/code_location.go#L25:
+// - simplified API and thus renamed (calls debug.Stack() instead of taking a parameter)
+// - source code filtering updated to be specific to Kubernetes
+// - optimized to use bytes and in-place slice filtering from
+//   https://github.com/golang/go/wiki/SliceTricks#filter-in-place
+func PrunedStack(skip int) []byte {
+	fullStackTrace := debug.Stack()
+	stack := bytes.Split(fullStackTrace, []byte("\n"))
+	// Ensure that the even entries are the method names and the
+	// the odd entries the source code information.
+	if len(stack) > 0 && bytes.HasPrefix(stack[0], []byte("goroutine ")) {
+		// Ignore "goroutine 29 [running]:" line.
+		stack = stack[1:]
+	}
+	// The "+2" is for skipping over:
+	// - runtime/debug.Stack()
+	// - PrunedStack()
+	skip += 2
+	if len(stack) > 2*skip {
+		stack = stack[2*skip:]
+	}
+	n := 0
+	for i := 0; i < len(stack)/2; i++ {
+		// We filter out based on the source code file name.
+		if !codeFilterRE.Match([]byte(stack[i*2+1])) {
+			stack[n] = stack[i*2]
+			stack[n+1] = stack[i*2+1]
+			n += 2
+		}
+	}
+	stack = stack[:n]
+
+	return bytes.Join(stack, []byte("\n"))
+}

test/e2e/framework/mockservers.go

@@ -0,0 +1,60 @@
+package framework
+
+import (
+	"github.com/fatedier/frp/test/e2e/mock/echoserver"
+	"github.com/fatedier/frp/test/e2e/pkg/port"
+)
+
+const (
+	TCPEchoServerPort = "TCPEchoServerPort"
+	UDPEchoServerPort = "UDPEchoServerPort"
+)
+
+type MockServers struct {
+	tcpEchoServer *echoserver.Server
+	udpEchoServer *echoserver.Server
+}
+
+func NewMockServers(portAllocator *port.Allocator) *MockServers {
+	s := &MockServers{}
+	tcpPort := portAllocator.Get()
+	udpPort := portAllocator.Get()
+	s.tcpEchoServer = echoserver.New(echoserver.Options{
+		Type:      echoserver.TCP,
+		BindAddr:  "127.0.0.1",
+		BindPort:  int32(tcpPort),
+		RepeatNum: 1,
+	})
+	s.udpEchoServer = echoserver.New(echoserver.Options{
+		Type:      echoserver.UDP,
+		BindAddr:  "127.0.0.1",
+		BindPort:  int32(udpPort),
+		RepeatNum: 1,
+	})
+	return s
+}
+
+func (m *MockServers) Run() error {
+	if err := m.tcpEchoServer.Run(); err != nil {
+		return err
+	}
+	if err := m.udpEchoServer.Run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *MockServers) GetTemplateParams() map[string]interface{} {
+	ret := make(map[string]interface{})
+	ret[TCPEchoServerPort] = m.tcpEchoServer.GetOptions().BindPort
+	ret[UDPEchoServerPort] = m.udpEchoServer.GetOptions().BindPort
+	return ret
+}
+
+func (m *MockServers) GetParam(key string) interface{} {
+	params := m.GetTemplateParams()
+	if v, ok := params[key]; ok {
+		return v
+	}
+	return nil
+}

test/e2e/framework/process.go

@@ -0,0 +1,59 @@
+package framework
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/pkg/process"
+	flog "github.com/fatedier/frp/utils/log"
+)
+
+func GenerateConfigFile(path string, content string) error {
+	return ioutil.WriteFile(path, []byte(content), 0666)
+}
+
+// RunProcesses run multiple processes from templates.
+// The first template should always be frps.
+func (f *Framework) RunProcesses(serverTemplates []string, clientTemplates []string) {
+	templates := make([]string, 0, len(serverTemplates)+len(clientTemplates))
+	for _, t := range serverTemplates {
+		templates = append(templates, t)
+	}
+	for _, t := range clientTemplates {
+		templates = append(templates, t)
+	}
+	outs, ports, err := f.RenderTemplates(templates)
+	ExpectNoError(err)
+	ExpectTrue(len(templates) > 0)
+
+	f.UsedPorts = ports
+
+	for i := range serverTemplates {
+		path := filepath.Join(f.TempDirectory, fmt.Sprintf("frp-e2e-server-%d", i))
+		err = ioutil.WriteFile(path, []byte(outs[i]), 0666)
+		ExpectNoError(err)
+		flog.Trace("[%s] %s", path, outs[i])
+		p := process.New(TestContext.FRPServerPath, []string{"-c", path})
+		f.serverConfPaths = append(f.serverConfPaths, path)
+		f.serverProcesses = append(f.serverProcesses, p)
+		err = p.Start()
+		ExpectNoError(err)
+		time.Sleep(500 * time.Millisecond)
+	}
+
+	for i := range clientTemplates {
+		index := i + len(serverTemplates)
+		path := filepath.Join(f.TempDirectory, fmt.Sprintf("frp-e2e-client-%d", i))
+		err = ioutil.WriteFile(path, []byte(outs[index]), 0666)
+		ExpectNoError(err)
+		flog.Trace("[%s] %s", path, outs[index])
+		p := process.New(TestContext.FRPClientPath, []string{"-c", path})
+		f.clientConfPaths = append(f.clientConfPaths, path)
+		f.clientProcesses = append(f.clientProcesses, p)
+		err = p.Start()
+		ExpectNoError(err)
+		time.Sleep(500 * time.Millisecond)
+	}
+}

test/e2e/framework/request.go

@@ -0,0 +1,13 @@
+package framework
+
+import (
+	"time"
+
+	"github.com/fatedier/frp/test/e2e/pkg/request"
+)
+
+func ExpectTCPReuqest(port int, in, out []byte, timeout time.Duration) {
+	res, err := request.SendTCPRequest(port, in, timeout)
+	ExpectNoError(err)
+	ExpectEqual(string(out), res)
+}

test/e2e/framework/test_context.go

@@ -0,0 +1,42 @@
+package framework
+
+import (
+	"flag"
+	"fmt"
+	"os"
+)
+
+type TestContextType struct {
+	FRPClientPath string
+	FRPServerPath string
+	LogLevel      string
+}
+
+var TestContext TestContextType
+
+// RegisterCommonFlags registers flags common to all e2e test suites.
+// The flag set can be flag.CommandLine (if desired) or a custom
+// flag set that then gets passed to viperconfig.ViperizeFlags.
+//
+// The other Register*Flags methods below can be used to add more
+// test-specific flags. However, those settings then get added
+// regardless whether the test is actually in the test suite.
+//
+func RegisterCommonFlags(flags *flag.FlagSet) {
+	flags.StringVar(&TestContext.FRPClientPath, "frpc-path", "../../bin/frpc", "The frp client binary to use.")
+	flags.StringVar(&TestContext.FRPServerPath, "frps-path", "../../bin/frps", "The frp server binary to use.")
+	flags.StringVar(&TestContext.LogLevel, "log-level", "debug", "Log level.")
+}
+
+func ValidateTestContext(t *TestContextType) error {
+	if t.FRPClientPath == "" || t.FRPServerPath == "" {
+		return fmt.Errorf("frpc and frps binary path can't be empty")
+	}
+	if _, err := os.Stat(t.FRPClientPath); err != nil {
+		return fmt.Errorf("load frpc-path error: %v", err)
+	}
+	if _, err := os.Stat(t.FRPServerPath); err != nil {
+		return fmt.Errorf("load frps-path error: %v", err)
+	}
+	return nil
+}

test/e2e/framework/util.go

@@ -0,0 +1,14 @@
+package framework
+
+import (
+	"github.com/google/uuid"
+)
+
+// RunID is a unique identifier of the e2e run.
+// Beware that this ID is not the same for all tests in the e2e run, because each Ginkgo node creates it separately.
+var RunID string
+
+func init() {
+	uuid, _ := uuid.NewUUID()
+	RunID = uuid.String()
+}