From c040ae4578cb54d48752285190e5992d6f22c27e Mon Sep 17 00:00:00 2001 From: guox191 Date: Thu, 21 Mar 2019 18:18:20 +0800 Subject: [PATCH 1/3] add test cases for compressed response --- lib/recorder.js | 4 +--- lib/webInterface.js | 6 ++--- test/server/server.js | 25 ++++++++++++++++---- test/spec_lib/proxyServerModule.js | 37 +++++++++++++++++++++++------- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/lib/recorder.js b/lib/recorder.js index 1a37739..e85052d 100644 --- a/lib/recorder.js +++ b/lib/recorder.js @@ -242,18 +242,16 @@ class Recorder extends events.EventEmitter { bodyContent = iconv.decode(bodyContent, currentCharset); } - result.mime = contentType; result.content = bodyContent.toString(); result.type = contentType && /application\/json/i.test(contentType) ? 'json' : 'text'; } else if (contentType && /image/i.test(contentType)) { result.type = 'image'; - result.mime = contentType; result.content = bodyContent; } else { result.type = contentType; - result.mime = contentType; result.content = bodyContent.toString(); } + result.mime = contentType; result.fileName = path.basename(record.path); result.statusCode = record.statusCode; } catch (e) { diff --git a/lib/webInterface.js b/lib/webInterface.js index f450221..1939e3e 100644 --- a/lib/webInterface.js +++ b/lib/webInterface.js @@ -146,10 +146,8 @@ class webInterface extends events.EventEmitter { if (err || !result) { res.json({}); } else if (result.statusCode === 200 && result.mime) { - if (result.type === 'json' || - result.mime.indexOf('text') === 0 || - // deal with 'application/x-javascript' and 'application/javascript' - result.mime.indexOf('javascript') > -1) { + // deal with 'application/x-javascript' and 'application/javascript' + if (/json|text|javascript/.test(result.mime)) { _resContent(); } else if (result.type === 'image') { _resDownload(false); diff --git a/test/server/server.js b/test/server/server.js index aae74b9..f122b9e 100644 --- a/test/server/server.js +++ b/test/server/server.js @@ -11,6 +11,9 @@ const color = require('colorful'); const WebSocketServer = require('ws').Server; const tls = require('tls'); const crypto = require('crypto'); +const stream = require('stream'); +const brotli = require('brotli'); +const zlib = require('zlib'); const createSecureContext = tls.createSecureContext || crypto.createSecureContext; @@ -209,11 +212,6 @@ KoaServer.prototype.constructRouter = function () { this.response.set('Allow', 'GET, HEAD, POST, OPTIONS'); }); - // router.connect('/test/connect', function *(next) { - // printLog('requesting connect /test/connect'); - // this.response.body = 'connect_established_body'; - // }); - router.get('/test/should_not_replace_option', this.logRequest, function *(next) { this.response.body = 'the_option_that_not_be_replaced'; }); @@ -249,6 +247,23 @@ KoaServer.prototype.constructRouter = function () { this.response.body = buf; }); + router.get('/test/brotli', this.logRequest, function *(next) { + this.status = 200; + this.response.set('Content-Encoding', 'br'); + this.response.set('Content-Type', 'application/json'); + const buf = new Buffer('{"type":"brotli","message":"This is a brotli encoding response, but it need to be a long string or the brotli module\'s compress result will be null"}'); + this.response.body = Buffer.from(brotli.compress(buf)); + }); + + router.get('/test/gzip', this.logRequest, function *(next) { + this.status = 200; + this.response.set('Content-Encoding', 'gzip'); + this.response.set('Content-Type', 'application/json'); + const bufStream = new stream.PassThrough(); + bufStream.end(new Buffer('{"type":"gzip","message":"This is a gzip encoding response"}')); + this.response.body = bufStream.pipe(zlib.createGzip()); + }); + return router; }; diff --git a/test/spec_lib/proxyServerModule.js b/test/spec_lib/proxyServerModule.js index 0b87a2c..77e6fc5 100644 --- a/test/spec_lib/proxyServerModule.js +++ b/test/spec_lib/proxyServerModule.js @@ -2,13 +2,14 @@ * test for rule replaceOption rule * */ -const ip = require('ip'); const AnyProxy = require('../../proxy'); -const { proxyGet, directGet } = require('../util/HttpUtil.js'); +const { + proxyGet, + directGet, + generateUrl, +} = require('../util/HttpUtil.js'); const Server = require('../server/server.js'); -const OUT_BOUND_IP = ip.address(); - describe('AnyProxy.proxyServer basic test', () => { it('should successfully start a proxy server', done => { const options = { @@ -67,17 +68,17 @@ describe('AnyProxy.proxyServer high order test', () => { expect(res && res.statusCode && res.statusCode === 200 && res.body.length > 300).toBe(true); done(); }) - .catch(done) + .catch(done); }); it('should work as expected for ip host', done => { // test if proxy server works - proxyGet(`https://${OUT_BOUND_IP}:3001/test`, {}, {}) + proxyGet(generateUrl('https', '/test'), {}, {}) .then(res => { expect(res && res.statusCode && res.statusCode === 200).toBe(true); done(); }) - .catch(done) + .catch(done); }); it('should start webinterface correctly', done => { @@ -87,6 +88,26 @@ describe('AnyProxy.proxyServer high order test', () => { expect(res && res.statusCode && res.statusCode === 200 && res.body.length > 300).toBe(true); done(); }) - .catch(done) + .catch(done); + }); + + it('should deal well with the gzip encoding compressed response', done => { + proxyGet(generateUrl('https', '/test/gzip'), {}, {}) + .then(res => { + expect(res && res.statusCode === 200).toBe(true); + expect(JSON.parse(res.body).type).toBe('gzip'); + done(); + }) + .catch(done); + }); + + it('should deal well with the brotli encoding compressed response', done => { + proxyGet(generateUrl('https', '/test/brotli'), {}, {}) + .then(res => { + expect(res && res.statusCode === 200).toBe(true); + expect(JSON.parse(res.body).type).toBe('brotli'); + done(); + }) + .catch(done); }); }); From 763bdc07a25fada3ffd89d86e5aea036bb69f381 Mon Sep 17 00:00:00 2001 From: guox191 Date: Thu, 21 Mar 2019 23:15:08 +0800 Subject: [PATCH 2/3] test curlify function --- test/jasmine.json | 3 ++- test/spec_web/curlUtil.js | 50 ++++++++++++++++++++++++++++++++++++++ web/src/common/curlUtil.js | 48 ++++++++++++++---------------------- 3 files changed, 70 insertions(+), 31 deletions(-) create mode 100644 test/spec_web/curlUtil.js diff --git a/test/jasmine.json b/test/jasmine.json index 057167a..f455f85 100644 --- a/test/jasmine.json +++ b/test/jasmine.json @@ -2,7 +2,8 @@ "spec_dir": "test", "spec_files": [ "spec_lib/*.js", - "spec_rule/*.js" + "spec_rule/*.js", + "spec_web/*.js" ], "helpers": [ "../node_modules/babel-register/lib/node.js", diff --git a/test/spec_web/curlUtil.js b/test/spec_web/curlUtil.js new file mode 100644 index 0000000..5d33898 --- /dev/null +++ b/test/spec_web/curlUtil.js @@ -0,0 +1,50 @@ +const { curlify } = require('../../web/src/common/curlUtil'); + +describe('Test the curlify function', () => { + it('request with headers', () => { + const requestDetail = { + method: 'POST', + url: 'https://localhost:3001/test', + reqHeader: { + 'via-proxy': 'true', + }, + }; + const result = 'curl \'https://localhost:3001/test\' -X POST -H \'via-proxy: true\''; + expect(curlify(requestDetail)).toBe(result); + }); + + it('request with JSON body', () => { + const requestDetail = { + method: 'POST', + url: 'https://localhost:3001/test', + reqHeader: { + 'content-type': 'application/json; charset=utf-8', + }, + reqBody: '{"action":1,"method":"test"}', + }; + const result = `curl '${requestDetail.url}' -X POST -H 'content-type: application/json; charset=utf-8' -d '${requestDetail.reqBody}'`; + expect(curlify(requestDetail)).toBe(result); + }); + + it('accpet gzip encoding with compressed flag', () => { + const requestDetail = { + method: 'GET', + url: 'https://localhost:3001/test', + reqHeader: { + Host: 'localhost', + 'Accept-Encoding': 'gzip', + }, + }; + const result = 'curl \'https://localhost:3001/test\' -H \'Host: localhost\' -H \'Accept-Encoding: gzip\' --compressed'; + expect(curlify(requestDetail)).toBe(result); + }); + + it('escape url character', () => { + const requestDetail = { + method: 'GET', + url: 'https://localhost:3001/test?a[]=1', + }; + const result = 'curl \'https://localhost:3001/test?a\\[\\]=1\''; + expect(curlify(requestDetail)).toBe(result); + }); +}); diff --git a/web/src/common/curlUtil.js b/web/src/common/curlUtil.js index 7372e06..86540a4 100644 --- a/web/src/common/curlUtil.js +++ b/web/src/common/curlUtil.js @@ -1,37 +1,25 @@ +export function curlify(recordDetail) { + const headers = { ...recordDetail.reqHeader }; + const acceptEncoding = headers['Accept-Encoding'] || headers['accept-encoding']; + // escape reserve character in url + const url = recordDetail.url.replace(/([\[\]])/g, '\\$1'); + const curlified = ['curl', `'${url}'`]; -export function curlify(recordDetail){ - let curlified = [] - let type = '' - let headers = { ...recordDetail.reqHeader } - curlified.push('curl') - curlified.push('-X', recordDetail.method) - curlified.push(`'${recordDetail.url}'`) - - if (headers) { - type = headers['Content-Type'] - delete headers['Accept-Encoding'] - - for(let k of Object.keys(headers)){ - let v = headers[k] - curlified.push('-H') - curlified.push(`'${k}: ${v}'`) - } + if (recordDetail.method.toUpperCase() !== 'GET') { + curlified.push('-X', recordDetail.method); } - if (recordDetail.reqBody){ + Object.keys(headers).forEach((key) => { + curlified.push('-H', `'${key}: ${headers[key]}'`); + }); - if(type === 'multipart/form-data' && recordDetail.method === 'POST') { - let formDataBody = recordDetail.reqBody.split('&') - - for(let data of formDataBody) { - curlified.push('-F') - curlified.push(data) - } - } else { - curlified.push('-d') - curlified.push(recordDetail.reqBody) - } + if (recordDetail.reqBody) { + curlified.push('-d', `'${recordDetail.reqBody}'`); } - return curlified.join(' ') + if (/deflate|gzip/.test(acceptEncoding)) { + curlified.push('--compressed'); + } + + return curlified.join(' '); } From 3a310b0c0e3326421331b444450f9b000ec63b67 Mon Sep 17 00:00:00 2001 From: guox191 Date: Sat, 23 Mar 2019 14:51:01 +0800 Subject: [PATCH 3/3] test deflate content --- lib/requestHandler.js | 4 ++-- test/server/server.js | 7 +++++++ test/spec_lib/proxyServerModule.js | 14 ++++++++++++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/requestHandler.js b/lib/requestHandler.js index c2aafd3..012a3ed 100644 --- a/lib/requestHandler.js +++ b/lib/requestHandler.js @@ -120,7 +120,7 @@ function fetchRemoteResponse(protocol, options, reqData, config) { // only do unzip when there is res data if (ifServerGzipped && originContentLen) { refactContentEncoding(); - zlib.gunzip(serverResData, (err, buff) => { // TODO test case to cover + zlib.gunzip(serverResData, (err, buff) => { if (err) { rejectParsing(err); } else { @@ -129,7 +129,7 @@ function fetchRemoteResponse(protocol, options, reqData, config) { }); } else if (isServerDeflated && originContentLen) { refactContentEncoding(); - zlib.inflateRaw(serverResData, (err, buff) => { // TODO test case to cover + zlib.inflateRaw(serverResData, (err, buff) => { if (err) { rejectParsing(err); } else { diff --git a/test/server/server.js b/test/server/server.js index f122b9e..1dff4e0 100644 --- a/test/server/server.js +++ b/test/server/server.js @@ -264,6 +264,13 @@ KoaServer.prototype.constructRouter = function () { this.response.body = bufStream.pipe(zlib.createGzip()); }); + router.get('/test/deflate', this.logRequest, function *(next) { + this.status = 200; + this.response.set('Content-Encoding', 'deflate'); + this.response.set('Content-Type', 'application/json'); + this.response.body = zlib.deflateRawSync('{"type":"deflate","message":"This is a deflate encoding response"}'); + }); + return router; }; diff --git a/test/spec_lib/proxyServerModule.js b/test/spec_lib/proxyServerModule.js index 77e6fc5..3a52ef2 100644 --- a/test/spec_lib/proxyServerModule.js +++ b/test/spec_lib/proxyServerModule.js @@ -91,7 +91,7 @@ describe('AnyProxy.proxyServer high order test', () => { .catch(done); }); - it('should deal well with the gzip encoding compressed response', done => { + it('should deal well with the gzip encoding response', done => { proxyGet(generateUrl('https', '/test/gzip'), {}, {}) .then(res => { expect(res && res.statusCode === 200).toBe(true); @@ -101,7 +101,17 @@ describe('AnyProxy.proxyServer high order test', () => { .catch(done); }); - it('should deal well with the brotli encoding compressed response', done => { + it('should deal well with the deflate encoding response', done => { + proxyGet(generateUrl('https', '/test/deflate'), {}, {}) + .then(res => { + expect(res && res.statusCode === 200).toBe(true); + expect(JSON.parse(res.body).type).toBe('deflate'); + done(); + }) + .catch(done); + }); + + it('should deal well with the brotli encoding response', done => { proxyGet(generateUrl('https', '/test/brotli'), {}, {}) .then(res => { expect(res && res.statusCode === 200).toBe(true);