Merge branch 'master' of git://github.com/guox191/anyproxy into guox191-master

lib/recorder.js

@@ -247,18 +247,16 @@ class Recorder extends events.EventEmitter {
                 bodyContent = iconv.decode(bodyContent, currentCharset);
               }
 
-              result.mime = contentType;
               result.content = bodyContent.toString();
               result.type = contentType && /application\/json/i.test(contentType) ? 'json' : 'text';
             } else if (contentType && /image/i.test(contentType)) {
               result.type = 'image';
-              result.mime = contentType;
               result.content = bodyContent;
             } else {
               result.type = contentType;
-              result.mime = contentType;
               result.content = bodyContent.toString();
             }
+            result.mime = contentType;
             result.fileName = path.basename(record.path);
             result.statusCode = record.statusCode;
           } catch (e) {

lib/requestHandler.js

@@ -120,7 +120,7 @@ function fetchRemoteResponse(protocol, options, reqData, config) {
             // only do unzip when there is res data
             if (ifServerGzipped && originContentLen) {
               refactContentEncoding();
-              zlib.gunzip(serverResData, (err, buff) => { // TODO test case to cover
+              zlib.gunzip(serverResData, (err, buff) => {
                 if (err) {
                   rejectParsing(err);
                 } else {
@@ -129,7 +129,7 @@ function fetchRemoteResponse(protocol, options, reqData, config) {
               });
             } else if (isServerDeflated && originContentLen) {
               refactContentEncoding();
-              zlib.inflateRaw(serverResData, (err, buff) => { // TODO test case to cover
+              zlib.inflateRaw(serverResData, (err, buff) => {
                 if (err) {
                   rejectParsing(err);
                 } else {

lib/webInterface.js

@@ -146,10 +146,8 @@ class webInterface extends events.EventEmitter {
           if (err || !result) {
             res.json({});
           } else if (result.statusCode === 200 && result.mime) {
-            if (result.type === 'json' ||
+            // deal with 'application/x-javascript' and 'application/javascript'
-              result.mime.indexOf('text') === 0 ||
+            if (/json|text|javascript/.test(result.mime)) {
-              // deal with 'application/x-javascript' and 'application/javascript'
-              result.mime.indexOf('javascript') > -1) {
               _resContent();
             } else if (result.type === 'image') {
               _resDownload(false);

test/jasmine.json

@@ -2,7 +2,8 @@
   "spec_dir": "test",
   "spec_files": [
     "spec_lib/*.js",
-    "spec_rule/*.js"
+    "spec_rule/*.js",
+    "spec_web/*.js"
   ],
   "helpers": [
      "../node_modules/babel-register/lib/node.js",

test/server/server.js

@@ -11,6 +11,9 @@ const color = require('colorful');
 const WebSocketServer = require('ws').Server;
 const tls = require('tls');
 const crypto = require('crypto');
+const stream = require('stream');
+const brotli = require('brotli');
+const zlib = require('zlib');
 
 const createSecureContext = tls.createSecureContext || crypto.createSecureContext;
 
@@ -209,11 +212,6 @@ KoaServer.prototype.constructRouter = function () {
     this.response.set('Allow', 'GET, HEAD, POST, OPTIONS');
   });
 
-  // router.connect('/test/connect', function *(next) {
-  //     printLog('requesting connect /test/connect');
-  //     this.response.body = 'connect_established_body';
-  // });
-
   router.get('/test/should_not_replace_option', this.logRequest, function *(next) {
     this.response.body = 'the_option_that_not_be_replaced';
   });
@@ -249,6 +247,30 @@ KoaServer.prototype.constructRouter = function () {
     this.response.body = buf;
   });
 
+  router.get('/test/brotli', this.logRequest, function *(next) {
+    this.status = 200;
+    this.response.set('Content-Encoding', 'br');
+    this.response.set('Content-Type', 'application/json');
+    const buf = new Buffer('{"type":"brotli","message":"This is a brotli encoding response, but it need to be a long string or the brotli module\'s compress result will be null"}');
+    this.response.body = Buffer.from(brotli.compress(buf));
+  });
+
+  router.get('/test/gzip', this.logRequest, function *(next) {
+    this.status = 200;
+    this.response.set('Content-Encoding', 'gzip');
+    this.response.set('Content-Type', 'application/json');
+    const bufStream = new stream.PassThrough();
+    bufStream.end(new Buffer('{"type":"gzip","message":"This is a gzip encoding response"}'));
+    this.response.body = bufStream.pipe(zlib.createGzip());
+  });
+
+  router.get('/test/deflate', this.logRequest, function *(next) {
+    this.status = 200;
+    this.response.set('Content-Encoding', 'deflate');
+    this.response.set('Content-Type', 'application/json');
+    this.response.body = zlib.deflateRawSync('{"type":"deflate","message":"This is a deflate encoding response"}');
+  });
+
   return router;
 };
 

test/spec_lib/proxyServerModule.js

@@ -2,13 +2,14 @@
 * test for rule replaceOption rule
 *
 */
-const ip = require('ip');
 const AnyProxy = require('../../proxy');
-const { proxyGet, directGet } = require('../util/HttpUtil.js');
+const {
+  proxyGet,
+  directGet,
+  generateUrl,
+} = require('../util/HttpUtil.js');
 const Server = require('../server/server.js');
 
-const OUT_BOUND_IP = ip.address();
-
 describe('AnyProxy.proxyServer basic test', () => {
   it('should successfully start a proxy server', done => {
     const options = {
@@ -67,17 +68,17 @@ describe('AnyProxy.proxyServer high order test', () => {
         expect(res && res.statusCode && res.statusCode === 200 && res.body.length > 300).toBe(true);
         done();
       })
-      .catch(done)
+      .catch(done);
   });
 
   it('should work as expected for ip host', done => {
     // test if proxy server works
-    proxyGet(`https://${OUT_BOUND_IP}:3001/test`, {}, {})
+    proxyGet(generateUrl('https', '/test'), {}, {})
       .then(res => {
         expect(res && res.statusCode && res.statusCode === 200).toBe(true);
         done();
       })
-      .catch(done)
+      .catch(done);
   });
 
   it('should start webinterface correctly', done => {
@@ -87,6 +88,36 @@ describe('AnyProxy.proxyServer high order test', () => {
         expect(res && res.statusCode && res.statusCode === 200 && res.body.length > 300).toBe(true);
         done();
       })
-      .catch(done)
+      .catch(done);
+  });
+
+  it('should deal well with the gzip encoding response', done => {
+    proxyGet(generateUrl('https', '/test/gzip'), {}, {})
+      .then(res => {
+        expect(res && res.statusCode === 200).toBe(true);
+        expect(JSON.parse(res.body).type).toBe('gzip');
+        done();
+      })
+      .catch(done);
+  });
+
+  it('should deal well with the deflate encoding response', done => {
+    proxyGet(generateUrl('https', '/test/deflate'), {}, {})
+      .then(res => {
+        expect(res && res.statusCode === 200).toBe(true);
+        expect(JSON.parse(res.body).type).toBe('deflate');
+        done();
+      })
+      .catch(done);
+  });
+
+  it('should deal well with the brotli encoding response', done => {
+    proxyGet(generateUrl('https', '/test/brotli'), {}, {})
+      .then(res => {
+        expect(res && res.statusCode === 200).toBe(true);
+        expect(JSON.parse(res.body).type).toBe('brotli');
+        done();
+      })
+      .catch(done);
   });
 });

test/spec_web/curlUtil.js

@@ -0,0 +1,50 @@
+const { curlify } = require('../../web/src/common/curlUtil');
+
+describe('Test the curlify function', () => {
+  it('request with headers', () => {
+    const requestDetail = {
+      method: 'POST',
+      url: 'https://localhost:3001/test',
+      reqHeader: {
+        'via-proxy': 'true',
+      },
+    };
+    const result = 'curl \'https://localhost:3001/test\' -X POST -H \'via-proxy: true\'';
+    expect(curlify(requestDetail)).toBe(result);
+  });
+
+  it('request with JSON body', () => {
+    const requestDetail = {
+      method: 'POST',
+      url: 'https://localhost:3001/test',
+      reqHeader: {
+        'content-type': 'application/json; charset=utf-8',
+      },
+      reqBody: '{"action":1,"method":"test"}',
+    };
+    const result = `curl '${requestDetail.url}' -X POST -H 'content-type: application/json; charset=utf-8' -d '${requestDetail.reqBody}'`;
+    expect(curlify(requestDetail)).toBe(result);
+  });
+
+  it('accpet gzip encoding with compressed flag', () => {
+    const requestDetail = {
+      method: 'GET',
+      url: 'https://localhost:3001/test',
+      reqHeader: {
+        Host: 'localhost',
+        'Accept-Encoding': 'gzip',
+      },
+    };
+    const result = 'curl \'https://localhost:3001/test\' -H \'Host: localhost\' -H \'Accept-Encoding: gzip\' --compressed';
+    expect(curlify(requestDetail)).toBe(result);
+  });
+
+  it('escape url character', () => {
+    const requestDetail = {
+      method: 'GET',
+      url: 'https://localhost:3001/test?a[]=1',
+    };
+    const result = 'curl \'https://localhost:3001/test?a\\[\\]=1\'';
+    expect(curlify(requestDetail)).toBe(result);
+  });
+});

web/src/common/curlUtil.js

@@ -1,37 +1,25 @@
+export function curlify(recordDetail) {
+  const headers = { ...recordDetail.reqHeader };
+  const acceptEncoding = headers['Accept-Encoding'] || headers['accept-encoding'];
+  // escape reserve character in url
+  const url = recordDetail.url.replace(/([\[\]])/g, '\\$1');
+  const curlified = ['curl', `'${url}'`];
 
-export function curlify(recordDetail){
+  if (recordDetail.method.toUpperCase() !== 'GET') {
-  let curlified = []
+    curlified.push('-X', recordDetail.method);
-  let type = ''
-  let headers = { ...recordDetail.reqHeader }
-  curlified.push('curl')
-  curlified.push('-X', recordDetail.method)
-  curlified.push(`'${recordDetail.url}'`)
-
-  if (headers) {
-    type = headers['Content-Type']
-    delete headers['Accept-Encoding']
-
-    for(let k of Object.keys(headers)){
-      let v = headers[k]
-      curlified.push('-H')
-      curlified.push(`'${k}: ${v}'`)
-    }
   }
 
-  if (recordDetail.reqBody){
+  Object.keys(headers).forEach((key) => {
+    curlified.push('-H', `'${key}: ${headers[key]}'`);
+  });
 
-    if(type === 'multipart/form-data' && recordDetail.method === 'POST') {
+  if (recordDetail.reqBody) {
-      let formDataBody = recordDetail.reqBody.split('&')
+    curlified.push('-d', `'${recordDetail.reqBody}'`);
-
-      for(let data of formDataBody) {
-        curlified.push('-F')
-        curlified.push(data)
-      }
-    } else {
-      curlified.push('-d')
-      curlified.push(recordDetail.reqBody)
-    }
   }
 
-  return curlified.join(' ')
+  if (/deflate|gzip/.test(acceptEncoding)) {
+    curlified.push('--compressed');
+  }
+
+  return curlified.join(' ');
 }