Add test cases for the proxy, and do some tiny fixes.

the fixes are:
1. add "content-type" in headers for when dealing with localresponse
2. make a more accurate tip for throttle rate when lower than 1
3. make the crtMgr funcionality a more independent one
4. uppercase the request header before sending it out

update the tip

test/util/CommonUtil.js

@@ -0,0 +1,153 @@
+/**
+*
+* The utility class for test
+*/
+const zlib = require('zlib');
+const color = require('colorful');
+
+/*
+* Compare whether tow object are equal
+*/
+function isObjectEqual (source = {} , target = {}, url = '') {
+    source = Object.assign({}, source);
+    target = Object.assign({}, target);
+    let isEqual = true;
+
+    for(const key in source) {
+        isEqual = isEqual && source[key] === target[key];
+
+        if (!isEqual) {
+            console.info('source object :', source);
+            console.info('target object :', target);
+            printError(`different key in isObjectEqual is: "${key}", source is "${source[key]}",
+                target is "${target[key]}" the url is ${url}`);
+            break;
+        }
+
+        delete source[key];
+        delete target[key];
+    }
+
+    for(const key in target) {
+        isEqual = isEqual && source[key] === target[key];
+
+        if (!isEqual) {
+            console.info('source object :', source);
+            console.info('target object :', target);
+            printError(`different key in isObjectEqual is: "${key}", source is "${source[key]}",
+                target is "${target[key]}" the url is ${url}`);
+            break;
+        }
+
+        delete source[key];
+        delete target[key];
+    }
+
+    return isEqual;
+}
+
+/*
+* Compare the header between direct with proxy
+* Will exclude the header(s) which modified by proxy
+*/
+function isCommonResHeaderEqual (directHeaders, proxyHeaders, requestUrl) {
+    directHeaders = Object.assign({}, directHeaders);
+    proxyHeaders = Object.assign({}, proxyHeaders);
+    let isEqual = true;
+    const mustEqualFileds = []; // the fileds that have to be equal, or the assert will be failed
+
+    if (!/gzip/i.test(directHeaders['content-encoding'])) {
+        // if the content is gzipped, proxy will unzip and remove the header
+        mustEqualFileds.push('content-encoding');
+        mustEqualFileds.push('content-length');
+    }
+    mustEqualFileds.push('content-type');
+    mustEqualFileds.push('cache-control');
+    mustEqualFileds.push('allow');
+
+    // ensure the required fileds are same
+    mustEqualFileds.forEach(filedName => {
+        isEqual = directHeaders[filedName] === proxyHeaders[filedName];
+        delete directHeaders[filedName];
+        delete proxyHeaders[filedName];
+    });
+
+    // remained filed are good to be same, but are allowed to be different
+    // will warn out those different fileds
+    for (const key in directHeaders) {
+        if (directHeaders[key] !== proxyHeaders[key]) {
+            printWarn(`key "${key}" of two response headers are different in request "${requestUrl}" :
+             direct is: "${directHeaders[key]}", proxy is: "${proxyHeaders[key]}"`);
+        }
+        continue;
+    }
+
+    return isEqual;
+}
+
+/*
+* Compare the request between direct with proxy
+*
+*/
+function isCommonReqEqual(url, serverInstance) {
+    try{
+        url = url.replace('https://', '').replace('http://', ''); // only the remained path is required
+        let isEqual = true;
+
+        const directReqObj = serverInstance.getRequestRecord(url);
+        const proxyReqObj = serverInstance.getProxyRequestRecord(url);
+
+        // ensure the proxy header is correct
+        isEqual = isEqual && proxyReqObj.headers['via-proxy'] === 'true';
+        delete proxyReqObj.headers['via-proxy'];
+
+        // exclued accept-encoding from comparing, since the proxy will remove it before sending it out
+        delete directReqObj.headers['accept-encoding'];
+
+        // per undefined header, proxy will set it with 0, and an empty request body
+        if (typeof directReqObj.headers['content-length'] === 'undefined') {
+
+            directReqObj.headers['content-length'] = "0";
+        }
+
+        directReqObj.headers['content-type'] = trimFormContentType(directReqObj.headers['content-type']);
+        proxyReqObj.headers['content-type'] = trimFormContentType(proxyReqObj.headers['content-type']);
+
+        isEqual = isEqual && directReqObj.url === proxyReqObj.url;
+        isEqual = isEqual && isObjectEqual(directReqObj.headers, proxyReqObj.headers, url);
+        isEqual = isEqual && directReqObj.body === proxyReqObj.body;
+        return isEqual;
+    } catch (e) {
+        console.error(e);
+    }
+
+}
+
+/*
+* for multipart-form, the boundary will be different with each update, we trim it here
+*/
+function trimFormContentType (contentType = '') {
+    return contentType.replace(/boundary.*/, '');
+}
+
+
+function printLog (content) {
+    console.log(color.blue('==LOG==: ' + content));
+}
+
+function printWarn(content) {
+    console.log(color.magenta('==WARN==: ' + content));
+}
+
+function printError(content) {
+    console.log(color.red('==ERROR==: ' + content));
+}
+
+module.exports = {
+    isObjectEqual,
+    isCommonResHeaderEqual,
+    printLog,
+    printWarn,
+    printError,
+    isCommonReqEqual
+};

test/util/HttpUtil.js

@@ -0,0 +1,262 @@
+/**
+ * An util to make the request out
+ *
+ */
+const querystring = require('querystring');
+const http = require('http');
+const zlib = require('zlib');
+const Buffer = require('buffer').Buffer;
+const request = require('request');
+const fs = require('fs');
+const WebSocket = require('ws');
+const HttpsProxyAgent = require('https-proxy-agent');
+
+const DEFAULT_HOST = 'localhost';
+const PROXY_HOST = 'http://localhost:8001';
+const SOCKET_PROXY_HOST = 'http://localhost:8001';
+
+
+const HTTP_SERVER_BASE = 'http://localhost:3000';
+const HTTPS_SERVER_BASE = 'https://localhost:3001';
+const WS_SERVER_BASE = 'ws://localhost:3000';
+const WSS_SERVER_BASE = 'wss://localhost:3001';
+
+const DEFAULT_PROXY_OPTIONS = {
+    port: 8001, // proxy的端口
+    method: 'GET',
+    host: 'localhost'
+};
+
+const DEFAULT_OPTIONS = {
+
+};
+
+function getHostFromUrl (url = '') {
+    const hostReg = /^(https{0,1}:\/\/)(\w+)/;
+    const match = url.match(hostReg);
+
+    return match && match[2] ? match[2] : '';
+}
+
+function getPortFromUrl (url = '') {
+    const portReg = /^https{0,1}:\/\/\w+(:(\d+)){0,1}/;
+    const match = url.match(portReg);
+    let port = match && match[2] ? match[2] : '';
+
+    if (!port) {
+        port = url.indexOf('https://') === 0 ? 443 : 80;
+    }
+    return port;
+}
+
+/**
+ * 获取url中的path
+ */
+function getPathFromUrl (url = '') {
+    const pathReg = /^https{0,1}:\/\/\w+(:\d+){0,1}(.+)/;
+    const match = url.match(pathReg);
+    const path = match && match[3] ? match[2] : url;
+    return path;
+}
+
+function proxyRequest (method = 'GET', url, params, headers = {}) {
+    return doRequest(method, url, params, headers, true);
+}
+
+/*
+ * 直接请求到真实服务器，不经过代理服务器
+ *
+ */
+function directRequest (method = 'GET', url, params, headers = {}) {
+    return doRequest(method, url, params, headers);
+}
+
+function directUpload (url, filepath, formParams = {}, headers = {}) {
+    return doUpload(url, 'POST', filepath, formParams, headers);
+}
+
+function proxyUpload (url, filepath, formParams = {}, headers = {}) {
+    return doUpload(url, 'POST', filepath, formParams, headers, true);
+}
+
+function directPutUpload (url, filepath, formParams = {}, headers = {}) {
+    return doUpload(url, 'PUT', filepath, formParams, headers);
+}
+
+function proxyPutUpload (url, filepath, headers = {}) {
+    return doUpload(url, 'PUT', filepath, headers, true);
+}
+
+function doRequest (method = 'GET', url, params, headers = {}, isProxy) {
+    headers = Object.assign({}, headers);
+    const requestData = {
+        method: method,
+        form: params,
+        url: url,
+        headers: headers,
+        rejectUnauthorized: false
+    };
+
+    if (isProxy) {
+        requestData.proxy = PROXY_HOST;
+        requestData.headers['via-proxy'] = 'true';
+    }
+
+    const requestTask = new Promise((resolve, reject) => {
+        request(
+            requestData,
+            function (error, response, body) {
+                if (error) {
+                    reject(error);
+                    return;
+                }
+                resolve(response);
+            }
+        );
+    });
+    return requestTask;
+}
+
+function doUpload (url, method, filepath, formParams, headers = {}, isProxy) {
+    let formData = {
+        file: fs.createReadStream(filepath)
+    };
+
+    formData = Object.assign({}, formData, formParams);
+    headers = Object.assign({}, headers);
+
+    const requestData = {
+        formData: formData,
+        url: url,
+        method: method,
+        headers: headers,
+        json: true,
+        rejectUnauthorized: false
+    };
+
+    if (isProxy) {
+        requestData.proxy = PROXY_HOST;
+        requestData.headers['via-proxy'] = 'true';
+    }
+    const requestTask = new Promise((resolve, reject) => {
+        request(
+            requestData,
+            function (error, response, body) {
+                if (error) {
+                    reject(error);
+                    return;
+                }
+                resolve(response);
+            }
+        );
+    });
+    return requestTask;
+}
+
+function doWebSocket(url, isProxy) {
+    let ws;
+    if (isProxy) {
+        const agent = new HttpsProxyAgent(SOCKET_PROXY_HOST);
+        ws = new WebSocket(url, {
+            agent: agent,
+            rejectUnauthorized: false
+        });
+    } else {
+        ws = new WebSocket(url, {
+            rejectUnauthorized: false
+        });
+    }
+
+    return ws;
+}
+
+function proxyGet (url, params, headers = {}) {
+    return proxyRequest('GET', url, params, headers);
+}
+
+function proxyPost (url, params, headers = {}) {
+    return proxyRequest('POST', url, params, headers);
+}
+
+function proxyPut (url, params, headers = {}) {
+    return proxyRequest('PUT', url, params, headers);
+}
+
+function proxyDelete (url, params, headers = {}) {
+    return proxyRequest('DELETE', url, params, headers);
+}
+
+function proxyHead(url, headers = {}) {
+    return proxyRequest('HEAD', url, {}, headers);
+}
+
+function proxyOptions(url, headers = {}) {
+    return proxyRequest('OPTIONS', url, {}, headers);
+}
+
+function directGet (url, params, headers = {}) {
+    return directRequest('GET', url, params, headers);
+}
+
+function directPost (url, params, headers = {}) {
+    return directRequest('POST', url, params, headers);
+}
+
+function directPut (url, params, headers = {}) {
+    return directRequest('PUT', url, params, headers);
+}
+
+function directDelete (url, params, headers = {}) {
+    return directRequest('DELETE', url, params, headers);
+}
+
+function directHead (url, headers = {}) {
+    return directRequest('HEAD', url, {} , headers);
+}
+
+function directOptions (url, headers ={}) {
+    return directRequest('OPTIONS', url,  {}, headers);
+}
+
+function proxyWs (url) {
+    return doWebSocket(url, true);
+}
+
+function directWs (url) {
+    return doWebSocket(url);
+}
+
+/**
+* generate the final url based on protocol and path
+*
+*/
+function generateUrl (protocol, urlPath) {
+    return protocol === 'http' ? HTTP_SERVER_BASE + urlPath : HTTPS_SERVER_BASE + urlPath;
+}
+
+function generateWsUrl (protocol, urlPath) {
+    return protocol === 'wss' ? WSS_SERVER_BASE + urlPath : WS_SERVER_BASE + urlPath;
+}
+
+module.exports = {
+    proxyGet,
+    proxyPost,
+    directGet,
+    directPost,
+    directUpload,
+    proxyUpload,
+    generateUrl,
+    proxyWs,
+    directWs,
+    generateWsUrl,
+    directPut,
+    proxyPut,
+    directDelete,
+    proxyDelete,
+    directHead,
+    proxyHead,
+    directOptions,
+    proxyOptions,
+    directPutUpload,
+    proxyPutUpload
+};

test/util/ProxyServerUtil.js

@@ -0,0 +1,65 @@
+/*
+* Utility class for creating proxy server, used to create specfied proxy server
+*
+*/
+
+let proxy = require('../../proxy.js');
+const util = require('../../lib/util.js');
+
+const DEFAULT_OPTIONS = {
+    type: "http",
+    port: 8001,
+    hostname: "localhost",
+    dbFile: null,  // optional, save request data to a specified file, will use in-memory db if not specified
+    webPort: 8002,  // optional, port for web interface
+    socketPort: 8003,  // optional, internal port for web socket, replace this when it is conflict with your own service
+    throttle: 10000,    // optional, speed limit in kb/s
+    disableWebInterface: false, //optional, set it when you don't want to use the web interface
+    setAsGlobalProxy: false, //set anyproxy as your system proxy
+    interceptHttps: true, // intercept https as well
+    silent: false //optional, do not print anything into terminal. do not set it when you are still debugging.
+};
+
+/**
+*
+* @return An instance of proxy, could be closed by calling `instance.close()`
+*/
+function defaultProxyServer () {
+    proxy = util.freshRequire('../proxy.js');
+
+    const options = util.merge({}, DEFAULT_OPTIONS);
+    options.rule = util.freshRequire('./rule_default.js');
+    return new proxy.proxyServer(options);
+}
+
+/*
+* Create proxy server with rule
+* @param rules
+    Object, the rule object which contains required intercept method
+  @return An instance of proxy, could be closed by calling `instance.close()`
+*/
+function proxyServerWithRule (rule) {
+    proxy = util.freshRequire('../proxy.js');
+
+    const options = util.merge({}, DEFAULT_OPTIONS);
+    options.rule = rule;
+
+    return new proxy.proxyServer(options);
+}
+
+function proxyServerWithoutHttpsIntercept (rule) {
+    proxy = util.freshRequire('../proxy.js');
+
+    const options = util.merge({}, DEFAULT_OPTIONS);
+    if (rule) {
+        options.rule = rule;
+    }
+    options.interceptHttps = false;
+    return new proxy.proxyServer(options);
+}
+
+module.exports = {
+    defaultProxyServer,
+    proxyServerWithoutHttpsIntercept,
+    proxyServerWithRule
+};