From 8ad15162fa7f9b14ae84d3a7d073ff7a09a2cf05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8A=A0=E9=87=8C?= <xiaofeng.mxf@taobao.com>
Date: Tue, 2 Sep 2014 14:54:45 +0800
Subject: [PATCH] update post body recorder
---
lib/recorder.js | 2 +-
lib/requestHandler.js | 137 ++++++++++--------
lib/rule_default.js | 9 +-
lib/util.js | 20 +++
package.json | 3 +-
proxy.js | 6 +
rule_sample/rule__blank.js | 40 +++--
rule_sample/rule_adjust_response_time.js | 26 +---
rule_sample/rule_allow_CORS.js | 23 +--
.../rule_intercept_some_https_requests.js | 20 ---
rule_sample/rule_remove_cache_header.js | 25 ----
rule_sample/rule_replace_request_option.js | 25 ----
rule_sample/rule_replace_response_data.js | 24 ---
.../rule_replace_response_status_code.js | 21 ---
rule_sample/rule_use_local_data.js | 33 +----
web/index.html | 7 +
web/page.js | 8 +-
17 files changed, 163 insertions(+), 266 deletions(-)
create mode 100644 lib/util.js
diff --git a/lib/recorder.js b/lib/recorder.js
index 5f65a45..8f204d1 100644
--- a/lib/recorder.js
+++ b/lib/recorder.js
@@ -82,6 +82,7 @@ function normalizeInfo(id,info){
//req
singleRecord.reqHeader = info.req.headers;
singleRecord.startTime = info.startTime;
+ singleRecord.reqBody = info.reqBody || "";
//res
if(info.endTime){
@@ -105,7 +106,6 @@ function normalizeInfo(id,info){
singleRecord.duration = "";
}
-
return singleRecord;
}
diff --git a/lib/requestHandler.js b/lib/requestHandler.js
index 16a2d4d..8c3c118 100644
--- a/lib/requestHandler.js
+++ b/lib/requestHandler.js
@@ -8,10 +8,12 @@ var http = require("http"),
async = require('async'),
color = require("colorful"),
Buffer = require('buffer').Buffer,
+ util = require("./util"),
httpsServerMgr = require("./httpsServerMgr");
var httpsServerMgrInstance = new httpsServerMgr(),
- userRule = require("./rule_default.js"); //default rule file
+ defaultRule = require("./rule_default.js"),
+ userRule = defaultRule; //init
function userRequestHandler(req,userRes){
var host = req.headers.host,
@@ -19,7 +21,8 @@ function userRequestHandler(req,userRes){
path = urlPattern.path,
protocol = (!!req.connection.encrypted && !/http:/.test(req.url)) ? "https" : "http",
resourceInfo,
- resourceInfoId = -1;
+ resourceInfoId = -1,
+ reqData;
//record
resourceInfo = {
@@ -30,84 +33,113 @@ function userRequestHandler(req,userRes){
req : req,
startTime : new Date().getTime()
};
-
- try{
+ if(GLOBAL.recorder){
resourceInfoId = GLOBAL.recorder.appendRecord(resourceInfo);
- }catch(e){}
+ }
console.log(color.green("\nreceived request to : " + host + path));
- /*
- req.url is wired
- in http server : http://www.example.com/a/b/c
- in https server : /work/alibaba
- */
- if(userRule.shouldUseLocalResponse(req)){
- console.log("==>use local rules");
- userRule.dealLocalResponse(req,function(statusCode,resHeader,resBody){
+ //get request body and route to local or remote
+ async.series([fetchReqData,routeReq],function(){});
+
+ //get request body
+ function fetchReqData(callback){
+ var postData = [];
+ req.on("data",function(chunk){
+ postData.push(chunk);
+ });
+ req.on("end",function(){
+ reqData = Buffer.concat(postData);
+ resourceInfo.reqBody = reqData.toString();
+ GLOBAL.recorder && GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
+
+ callback();
+ });
+ }
+
+ //route to dealing function
+ function routeReq(callback){
+ if(userRule.shouldUseLocalResponse(req,reqData)){
+ console.log("==>use local rules");
+ dealWithLocalResponse(callback);
+ }else{
+ console.log("==>will forward to real server by proxy");
+ dealWithRemoteResonse(callback);
+ }
+ }
+
+ function dealWithLocalResponse(callback){
+ userRule.dealLocalResponse(req,reqData,function(statusCode,resHeader,resBody){
//update record info
resourceInfo.endTime = new Date().getTime();
resourceInfo.res = { //construct a self-defined res object
statusCode : statusCode || "",
- headers : resHeader || {}
+ headers : resHeader || {}
}
- resourceInfo.resBody = resBody;
- resourceInfo.length = resBody.length;
-
- try{
- GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
- }catch(e){}
+ resourceInfo.resHeader = resHeader || {};
+ resourceInfo.resBody = resBody;
+ resourceInfo.length = resBody.length;
+ resourceInfo.statusCode = statusCode;
+
+ GLOBAL.recorder && GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
userRes.writeHead(statusCode,resHeader);
userRes.end(resBody);
+ callback && callback();
});
return;
+ }
- }else{
- console.log("==>will forward to real server by proxy");
+ function dealWithRemoteResonse(callback){
+ var options;
- //modify protocol if needed
+ //modify request protocol
protocol = userRule.replaceRequestProtocol(req,protocol) || protocol;
- var options = {
+ //modify request options
+ options = {
hostname : urlPattern.hostname || req.headers.host,
port : urlPattern.port || req.port || (/https/.test(protocol) ? 443 : 80),
path : path,
method : req.method,
headers : req.headers
};
-
- //modify request options
options = userRule.replaceRequestOption(req,options) || options;
+ //update quest data
+ reqData = userRule.replaceRequestData(req,reqData) || reqData;
+ options.headers = util.lower_keys(options.headers);
+ options.headers["content-length"] = reqData.length; //rewrite content length info
+
+ //send request
var proxyReq = ( /https/.test(protocol) ? https : http).request(options, function(res) {
+
+ //deal response header
var statusCode = res.statusCode;
statusCode = userRule.replaceResponseStatusCode(req,res,statusCode) || statusCode;
var resHeader = userRule.replaceResponseHeader(req,res,res.headers) || res.headers;
- resHeader = lower_keys(resHeader);
+ resHeader = util.lower_keys(resHeader);
- /* remove gzip related header, and ungzip the content */
+ // remove gzip related header, and ungzip the content
var ifServerGzipped = /gzip/i.test(resHeader['content-encoding']);
delete resHeader['content-encoding'];
delete resHeader['content-length'];
userRes.writeHead(statusCode, resHeader);
- //waiting for data
- var resData = [],
- length;
+ //deal response data
+ var length,
+ resData = [];
res.on("data",function(chunk){
resData.push(chunk);
});
res.on("end",function(){
-
- var serverResData,
- userCustomResData;
+ var serverResData;
async.series([
@@ -125,9 +157,7 @@ function userRequestHandler(req,userRes){
//get custom response
},function(callback){
-
- userCustomResData = userRule.replaceServerResData(req,res,serverResData);
- serverResData = userCustomResData || serverResData;
+ serverResData = userRule.replaceServerResData(req,res,serverResData) || serverResData;
callback();
//delay
@@ -152,15 +182,13 @@ function userRequestHandler(req,userRes){
resourceInfo.resBody = serverResData;
resourceInfo.length = serverResData.length;
- try{
- GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
- }catch(e){}
+ GLOBAL.recorder && GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
callback();
}
],function(err,result){
-
+ callback && callback();
});
});
@@ -175,7 +203,7 @@ function userRequestHandler(req,userRes){
userRes.end();
});
- req.pipe(proxyReq);
+ proxyReq.end(reqData);
}
}
@@ -255,9 +283,7 @@ function connectReqHandler(req, socket, head){
resourceInfo.resBody = "";
resourceInfo.length = 0;
- try{
- GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
- }catch(e){}
+ GLOBAL.recorder && GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
callback();
}
@@ -269,26 +295,21 @@ function connectReqHandler(req, socket, head){
});
}
-// {"Content-Encoding":"gzip"} --> {"content-encoding":"gzip"}
-function lower_keys(obj){
- for(var key in obj){
- var val = obj[key];
- delete obj[key];
-
- obj[key.toLowerCase()] = val;
- }
-
- return obj;
-}
-
function setRules(newRule){
if(!newRule){
return;
}else{
- userRule = newRule;
+ userRule = util.merge(defaultRule,newRule);
}
}
module.exports.userRequestHandler = userRequestHandler;
module.exports.connectReqHandler = connectReqHandler;
module.exports.setRules = setRules;
+
+/*
+note
+ req.url is wired
+ in http server : http://www.example.com/a/b/c
+ in https server : /work/alibaba
+*/
diff --git a/lib/rule_default.js b/lib/rule_default.js
index b80d8c6..75e63e4 100644
--- a/lib/rule_default.js
+++ b/lib/rule_default.js
@@ -1,14 +1,17 @@
module.exports = {
- shouldUseLocalResponse : function(req){
+ shouldUseLocalResponse : function(req,reqBody){
},
- dealLocalResponse : function(req,callback){
+ dealLocalResponse : function(req,reqBody,callback){
+ },
+
+ replaceRequestProtocol:function(req,protocol){
},
replaceRequestOption : function(req,option){
},
- replaceRequestProtocol:function(req,protocol){
+ replaceRequestData: function(req,data){
},
replaceResponseStatusCode: function(req,res,statusCode){
diff --git a/lib/util.js b/lib/util.js
new file mode 100644
index 0000000..8f58988
--- /dev/null
+++ b/lib/util.js
@@ -0,0 +1,20 @@
+// {"Content-Encoding":"gzip"} --> {"content-encoding":"gzip"}
+module.exports.lower_keys = function(obj){
+ for(var key in obj){
+ var val = obj[key];
+ delete obj[key];
+
+ obj[key.toLowerCase()] = val;
+ }
+
+ return obj;
+}
+
+module.exports.merge = function(baseObj, extendObj){
+ for(var key in extendObj){
+ baseObj[key] = extendObj[key];
+ }
+
+ return baseObj;
+}
+
diff --git a/package.json b/package.json
index 239e1d9..c9c82e3 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "anyproxy",
- "version": "2.0.0",
+ "version": "2.1.0",
"description": "a charles/fiddler like proxy written in NodeJs, which can handle HTTPS requests and CROS perfectly.",
"main": "proxy.js",
"bin": {
@@ -13,6 +13,7 @@
"commander": "~2.3.0",
"entities": "^1.1.1",
"express": "^4.8.5",
+ "iconv-lite": "^0.4.4",
"nedb": "^0.11.0",
"ws": "^0.4.32"
},
diff --git a/proxy.js b/proxy.js
index 7e95430..9072cf1 100644
--- a/proxy.js
+++ b/proxy.js
@@ -15,6 +15,12 @@ var http = require('http'),
GLOBAL.recorder = new Recorder();
+//mix some modules to global.util
+try{
+ GLOBAL.util = {};
+ GLOBAL.util['iconv-lite'] = require("iconv-lite");
+}catch(e){}
+
var T_TYPE_HTTP = 0,
T_TYPE_HTTPS = 1,
DEFAULT_PORT = 8001,
diff --git a/rule_sample/rule__blank.js b/rule_sample/rule__blank.js
index 7886edb..84af8f0 100644
--- a/rule_sample/rule__blank.js
+++ b/rule_sample/rule__blank.js
@@ -1,31 +1,27 @@
module.exports = {
/*
- these functions are required
- you may leave their bodies blank if necessary
+ these functions will overwrite the default ones, write your own when necessary.
*/
//whether to intercept this request by local logic
//if the return value is true, anyproxy will call dealLocalResponse to get response data and will not send request to remote server anymore
- shouldUseLocalResponse : function(req){
- return false;
+ shouldUseLocalResponse : function(req,reqBody){
+ if(/hello/.test(reqBody.toString())){
+ return true;
+ }else{
+ return false;
+ }
},
//you may deal the response locally instead of sending it to server
//this function be called when shouldUseLocalResponse returns true
//callback(statusCode,resHeader,responseData)
//e.g. callback(200,{"content-type":"text/html"},"hello world")
- dealLocalResponse : function(req,callback){
+ dealLocalResponse : function(req,reqBody,callback){
+ callback(200,{"content-type":"text/html"},reqBody);
//callback(statusCode,resHeader,responseData)
},
- //req is user's request sent to the proxy server
- // option is how the proxy server will send request to the real server. i.e. require("http").request(option,function(){...})
- //you may return a customized option to replace the original option
- replaceRequestOption : function(req,option){
- var newOption = option;
- return newOption;
- },
-
//replace the request protocol when sending to the real server
//protocol : "http" or "https"
replaceRequestProtocol:function(req,protocol){
@@ -33,6 +29,24 @@ module.exports = {
return newProtocol;
},
+ //req is user's request sent to the proxy server
+ //option is how the proxy server will send request to the real server. i.e. require("http").request(option,function(){...})
+ //you may return a customized option to replace the original option
+ //you should not write content-length header in options, since anyproxy will handle it for you
+ replaceRequestOption : function(req,option){
+ var newOption = option;
+ return newOption;
+ },
+
+ //replace the request body
+ replaceRequestData: function(req,data){
+ // console.log(data.toString().indexOf("alipay.acquire.order.precreate"));
+ // if(data.toString().indexOf("alipay.acquire.order.precreate") >= 0){
+ // req.needReplaceResponse = true;
+ // }
+ // return text;
+ },
+
//replace the statusCode before it's sent to the user
replaceResponseStatusCode: function(req,res,statusCode){
var newStatusCode = statusCode;
diff --git a/rule_sample/rule_adjust_response_time.js b/rule_sample/rule_adjust_response_time.js
index 6ccb6ab..32ec584 100644
--- a/rule_sample/rule_adjust_response_time.js
+++ b/rule_sample/rule_adjust_response_time.js
@@ -1,34 +1,10 @@
//rule scheme :
module.exports = {
- shouldUseLocalResponse : function(req){
- },
-
- dealLocalResponse : function(req,callback){
- },
-
- replaceRequestOption : function(req,option){
-
- },
-
- replaceRequestProtocol:function(req,protocol){
- },
-
- replaceResponseStatusCode: function(req,res,statusCode){
- },
-
- replaceResponseHeader: function(req,res,header){
- },
-
- replaceServerResData: function(req,res,serverResData){
-
- },
pauseBeforeSendingResponse : function(req,res){
//delay all the response for 1500ms
return 1500;
- },
-
- shouldInterceptHttpsReq :function(req){
}
+
};
\ No newline at end of file
diff --git a/rule_sample/rule_allow_CORS.js b/rule_sample/rule_allow_CORS.js
index e1f468f..182152c 100644
--- a/rule_sample/rule_allow_CORS.js
+++ b/rule_sample/rule_allow_CORS.js
@@ -2,7 +2,7 @@
// Ref: https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS
module.exports = {
- shouldUseLocalResponse : function(req){
+ shouldUseLocalResponse : function(req,reqBody){
//intercept all options request
if(req.method == "OPTIONS"){
return true;
@@ -11,33 +11,16 @@ module.exports = {
}
},
- dealLocalResponse : function(req,callback){
+ dealLocalResponse : function(req,reqBody,callback){
if(req.method == "OPTIONS"){
callback(200,mergeCORSHeader(req.headers),"");
}
},
- replaceRequestOption : function(req,option){
- },
-
- replaceRequestProtocol:function(req,protocol){
- },
-
- replaceResponseStatusCode: function(req,res,statusCode){
- },
-
replaceResponseHeader: function(req,res,header){
return mergeCORSHeader(req.headers, header);
- },
-
- replaceServerResData: function(req,res,serverResData){
- },
-
- pauseBeforeSendingResponse : function(req,res){
- },
-
- shouldInterceptHttpsReq :function(req){
}
+
};
function mergeCORSHeader(reqHeader,originHeader){
diff --git a/rule_sample/rule_intercept_some_https_requests.js b/rule_sample/rule_intercept_some_https_requests.js
index 3a3b94f..5ee36fc 100644
--- a/rule_sample/rule_intercept_some_https_requests.js
+++ b/rule_sample/rule_intercept_some_https_requests.js
@@ -1,24 +1,7 @@
//rule scheme :
module.exports = {
- shouldUseLocalResponse : function(req){
- },
- dealLocalResponse : function(req,callback){
- },
-
- replaceRequestOption : function(req,option){
-
- },
-
- replaceRequestProtocol:function(req,protocol){
- },
-
- replaceResponseStatusCode: function(req,res,statusCode){
- },
-
- replaceResponseHeader: function(req,res,header){
- },
replaceServerResData: function(req,res,serverResData){
//add "hello github" to all github pages
@@ -28,9 +11,6 @@ module.exports = {
return serverResData;
},
- pauseBeforeSendingResponse : function(req,res){
- },
-
shouldInterceptHttpsReq :function(req){
//intercept https://github.com/
//otherwise, all the https traffic will not go through this proxy
diff --git a/rule_sample/rule_remove_cache_header.js b/rule_sample/rule_remove_cache_header.js
index 12bbc34..5c93e7e 100644
--- a/rule_sample/rule_remove_cache_header.js
+++ b/rule_sample/rule_remove_cache_header.js
@@ -1,21 +1,6 @@
//rule scheme :
module.exports = {
- shouldUseLocalResponse : function(req){
- },
-
- dealLocalResponse : function(req,callback){
- },
-
- replaceRequestOption : function(req,option){
- },
-
- replaceRequestProtocol:function(req,protocol){
- },
-
- replaceResponseStatusCode: function(req,res,statusCode){
- },
-
replaceResponseHeader: function(req,res,header){
header = header || {};
header["Cache-Control"] = "no-cache, no-store, must-revalidate";
@@ -23,17 +8,7 @@ module.exports = {
header["Expires"] = 0;
return header;
- },
-
- replaceServerResData: function(req,res,serverResData){
- },
-
- pauseBeforeSendingResponse : function(req,res){
- },
-
- shouldInterceptHttpsReq :function(req){
}
-
};
function disableCacheHeader(header){
diff --git a/rule_sample/rule_replace_request_option.js b/rule_sample/rule_replace_request_option.js
index 268541d..6f3930c 100644
--- a/rule_sample/rule_replace_request_option.js
+++ b/rule_sample/rule_replace_request_option.js
@@ -1,11 +1,6 @@
//rule scheme :
module.exports = {
- shouldUseLocalResponse : function(req){
- },
-
- dealLocalResponse : function(req,callback){
- },
replaceRequestOption : function(req,option){
//replace request towards http://www.taobao.com
@@ -24,25 +19,5 @@ module.exports = {
if(option.hostname == "www.taobao.com" && option.path == "/"){
option.path = "/about/";
}
-
- console.log(option);
- },
-
- replaceRequestProtocol:function(req,protocol){
- },
-
- replaceResponseStatusCode: function(req,res,statusCode){
- },
-
- replaceResponseHeader: function(req,res,header){
- },
-
- replaceServerResData: function(req,res,serverResData){
- },
-
- pauseBeforeSendingResponse : function(req,res){
- },
-
- shouldInterceptHttpsReq :function(req){
}
};
\ No newline at end of file
diff --git a/rule_sample/rule_replace_response_data.js b/rule_sample/rule_replace_response_data.js
index daf4d69..872f5f4 100644
--- a/rule_sample/rule_replace_response_data.js
+++ b/rule_sample/rule_replace_response_data.js
@@ -1,24 +1,6 @@
//rule scheme :
module.exports = {
- shouldUseLocalResponse : function(req){
- },
-
- dealLocalResponse : function(req,callback){
- },
-
- replaceRequestOption : function(req,option){
-
- },
-
- replaceRequestProtocol:function(req,protocol){
- },
-
- replaceResponseStatusCode: function(req,res,statusCode){
- },
-
- replaceResponseHeader: function(req,res,header){
- },
replaceServerResData: function(req,res,serverResData){
@@ -31,11 +13,5 @@ module.exports = {
return serverResData;
}
- },
-
- pauseBeforeSendingResponse : function(req,res){
- },
-
- shouldInterceptHttpsReq :function(req){
}
};
\ No newline at end of file
diff --git a/rule_sample/rule_replace_response_status_code.js b/rule_sample/rule_replace_response_status_code.js
index 763bb1f..24b7777 100644
--- a/rule_sample/rule_replace_response_status_code.js
+++ b/rule_sample/rule_replace_response_status_code.js
@@ -1,18 +1,6 @@
//rule scheme :
module.exports = {
- shouldUseLocalResponse : function(req){
- },
-
- dealLocalResponse : function(req,callback){
- },
-
- replaceRequestOption : function(req,option){
-
- },
-
- replaceRequestProtocol:function(req,protocol){
- },
replaceResponseStatusCode: function(req,res,statusCode){
//redirect requests toward http://www.taobao.com/*
@@ -32,14 +20,5 @@ module.exports = {
}
return header;
- },
-
- replaceServerResData: function(req,res,serverResData){
- },
-
- pauseBeforeSendingResponse : function(req,res){
- },
-
- shouldInterceptHttpsReq :function(req){
}
};
\ No newline at end of file
diff --git a/rule_sample/rule_use_local_data.js b/rule_sample/rule_use_local_data.js
index 28a5520..a9221c8 100644
--- a/rule_sample/rule_use_local_data.js
+++ b/rule_sample/rule_use_local_data.js
@@ -1,9 +1,9 @@
//replace all the images with local one
-var url = require("url"),
- path = require("path"),
- fs = require("fs"),
- buffer = require("buffer");
+var url = require("url"),
+ path = require("path"),
+ fs = require("fs"),
+ buffer = require("buffer");
var map = [
{
@@ -15,7 +15,7 @@ var map = [
];
module.exports = {
- shouldUseLocalResponse : function(req){
+ shouldUseLocalResponse : function(req,reqBody){
var host = req.headers.host,
urlPattern = url.parse(req.url),
path = urlPattern.path;
@@ -45,31 +45,10 @@ module.exports = {
return false;
},
- dealLocalResponse : function(req,callback){
+ dealLocalResponse : function(req,reqBody,callback){
if(req.replaceLocalFile){
callback(200, {"content-type":"image/png"}, fs.readFileSync(req.replaceLocalFile) );
}
- },
-
- replaceRequestOption : function(req,option){
- },
-
- replaceRequestProtocol:function(req,protocol){
- },
-
- replaceResponseStatusCode: function(req,res,statusCode){
- },
-
- replaceResponseHeader: function(req,res,header){
- },
-
- replaceServerResData: function(req,res,serverResData){
- },
-
- pauseBeforeSendingResponse : function(req,res){
- },
-
- shouldInterceptHttpsReq :function(req){
}
};
diff --git a/web/index.html b/web/index.html
index 692549f..5a3c8ce 100644
--- a/web/index.html
+++ b/web/index.html
@@ -58,6 +58,13 @@
</div>
</section>
+ <section class="reqBody">
+ <h4 class="subTitle">request body</h4>
+ <div class="detail">
+ <p><%= reqBody %></p>
+ </div>
+ </section>
+
<% if(statusCode) { %>
<section class="resHeader">
<h4 class="subTitle">response header</h4>
diff --git a/web/page.js b/web/page.js
index ccbb455..366fe9c 100644
--- a/web/page.js
+++ b/web/page.js
@@ -154,10 +154,12 @@ seajs.use(['$','Underscore' ,'Backbone'], function($, _, Backbone) {
}
//data via web socket
- var dataSocket = new WebSocket("ws://127.0.0.1:8003");
- dataSocket.onopen = function(){
- console.log("dataSocket open");
+ if(!WebSocket){
+ alert("WebSocket is required. Please use a modern browser.");
+ return;
}
+ var dataSocket = new WebSocket("ws://127.0.0.1:8003");
+ dataSocket.onopen = function(){}
dataSocket.onmessage = function(event){
var data = JSON.parse(event.data);