XiaoMo/anyproxy
1
0
Fork 0
mirror of https://github.com/alibaba/anyproxy.git synced 2025-07-27 07:45:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

1.fix record issue with https 2.rewrite rule scheme

Browse Source
This commit is contained in:
加里
2014-09-01 16:38:43 +08:00
parent 0f4c1ac017
commit 48fb1cbe40
20 changed files with 633 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
lib/recorder.js
View File

@@ -45,16 +45,8 @@ function Recorder(){
if(!id || !info.resBody) return;
//add to body map
//do not save image data
if(/image/.test(info.res.headers['content-type'])){
if(/image/.test(info.resHeader['content-type'])){
self.recordBodyMap[id] = "(image)";
}else if(/gzip/.test(info.res.headers['content-encoding'])){
zlib.unzip(info.resBody,function(err,buffer){
if(err){
self.recordBodyMap[id] = "(err when unzip response buffer)";
}else{
self.recordBodyMap[id] = buffer.toString();
}
});
}else{
self.recordBodyMap[id] = info.resBody.toString();
}
@@ -81,7 +73,7 @@ function normalizeInfo(id,info){
//general
singleRecord._id = id;
singleRecord.id = id;
singleRecord.id = id;
singleRecord.url = info.url;
singleRecord.host = info.host;
singleRecord.path = info.path;
@@ -92,13 +84,13 @@ function normalizeInfo(id,info){
singleRecord.startTime = info.startTime;
//res
if(info.res){
singleRecord.statusCode= info.res.statusCode;
if(info.endTime){
singleRecord.statusCode= info.statusCode;
singleRecord.endTime = info.endTime;
singleRecord.resHeader = info.res.headers;
singleRecord.resHeader = info.resHeader;
singleRecord.length = info.length;
if(info.res.headers['content-type']){
singleRecord.mime = info.res.headers['content-type'].split(";")[0];
if(info.resHeader['content-type']){
singleRecord.mime = info.resHeader['content-type'].split(";")[0];
}else{
singleRecord.mime = "";
}

244
lib/requestHandler.js
View File

@@ -8,43 +8,28 @@ var http = require("http"),
async = require('async'),
color = require("colorful"),
Buffer = require('buffer').Buffer,
httpsServerMgr = require("./httpsServerMgr"),
userRule = require("./rule.js"); //TODO - to be configurable
httpsServerMgr = require("./httpsServerMgr");
var httpsServerMgrInstance = new httpsServerMgr();
//default rule
var handleRule = {
map :[
// {
// host :".",
// path :"/path/test",
// localFile :"",
// localDir :"~/"
// }
]
,httpsConfig:{
bypassAll : true,
interceptDomains:["^.*alibaba-inc\.com$"]
}
};
var httpsServerMgrInstance = new httpsServerMgr(),
userRule = require("./rule_default.js"); //default rule file
function userRequestHandler(req,userRes){
var host = req.headers.host,
var host = req.headers.host,
urlPattern = url.parse(req.url),
path = urlPattern.path,
callback = null, //TODO : remove callback
protocol = (!!req.connection.encrypted && !/http:/.test(req.url)) ? "https" : "http",
resourceInfo = {},
resourceInfo,
resourceInfoId = -1;
//record
resourceInfo.host = host;
resourceInfo.method = req.method;
resourceInfo.path = path;
resourceInfo.url = protocol + "://" + host + path;
resourceInfo.req = req;
resourceInfo.startTime = new Date().getTime();
resourceInfo = {
host : host,
method : req.method,
path : path,
url : protocol + "://" + host + path,
req : req,
startTime : new Date().getTime()
};
try{
resourceInfoId = GLOBAL.recorder.appendRecord(resourceInfo);
@@ -101,14 +86,17 @@ function userRequestHandler(req,userRes){
var statusCode = res.statusCode;
statusCode = userRule.replaceResponseStatusCode(req,res,statusCode) || statusCode;
var resHeader = res.headers;
resHeader = userRule.replaceResponseHeader(req,res,resHeader) || resHeader;
var resHeader = userRule.replaceResponseHeader(req,res,res.headers) || res.headers;
resHeader = lower_keys(resHeader);
//remove content-encoding
// delete resHeader['content-encoding'];
/* remove gzip related header, and ungzip the content */
var ifServerGzipped = /gzip/i.test(resHeader['content-encoding']);
delete resHeader['content-encoding'];
delete resHeader['content-length'];
userRes.writeHead(statusCode, resHeader);
//waiting for data
var resData = [],
length;
@@ -122,12 +110,11 @@ function userRequestHandler(req,userRes){
userCustomResData;
async.series([
//TODO : manage gzip
//unzip server res
//ungzip server res
function(callback){
serverResData = Buffer.concat(resData);
if(/gzip/i.test(res.headers['content-encoding'])){
if(ifServerGzipped ){
zlib.gunzip(serverResData,function(err,buff){
serverResData = buff;
callback();
@@ -140,20 +127,6 @@ function userRequestHandler(req,userRes){
},function(callback){
userCustomResData = userRule.replaceServerResData(req,res,serverResData);
//gzip users' string if necessary
if(typeof userCustomResData == "string" && /gzip/i.test(res.headers['content-encoding']) ){
zlib.gzip(userCustomResData,function(err,data){
userCustomResData = data;
console.log(data);
callback();
});
}else{
callback();
}
//generate response data
},function(callback){
serverResData = userCustomResData || serverResData;
callback();
@@ -168,18 +141,17 @@ function userRequestHandler(req,userRes){
//send response
},function(callback){
userRes.write(serverResData);
userRes.end();
userRes.end(serverResData);
callback();
//udpate record info
},function(callback){
resourceInfo.endTime = new Date().getTime();
resourceInfo.res = res; //TODO : replace res header / statusCode ?
resourceInfo.resBody = serverResData;
resourceInfo.length = serverResData.length;
resourceInfo.endTime = new Date().getTime();
resourceInfo.statusCode = statusCode;
resourceInfo.resHeader = resHeader;
resourceInfo.resBody = serverResData;
resourceInfo.length = serverResData.length;
try{
GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
}catch(e){}
@@ -208,80 +180,112 @@ function userRequestHandler(req,userRes){
}
function connectReqHandler(req, socket, head){
var hostname = req.url.split(":")[0],
var host = req.url.split(":")[0],
targetPort= req.url.split(":")[1],
httpsRule = handleRule.httpsConfig;
resourceInfo,
resourceInfoId;
var shouldBypass = !!httpsRule.bypassAll;
if(!shouldBypass){ //read rules
shouldBypass = true;
for(var index in httpsRule.interceptDomains){
var reg = new RegExp(httpsRule.interceptDomains[index]);
if( reg.test(hostname) ){
shouldBypass = false;
break;
}
}
}
console.log(color.green("\nreceived https CONNECT request " + hostname));
if(shouldBypass){
console.log("==>will bypass the man-in-the-middle proxy");
try{
var conn = net.connect(targetPort, hostname, function(){
socket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', function() {
conn.pipe(socket);
socket.pipe(conn);
});
});
conn.on("error",function(e){
console.log("err when connect to __host".replace(/__host/,hostname));
});
}catch(e){
console.log("err when connect to remote https server (__hostname)".replace(/__hostname/,hostname));//TODO
}
var shouldIntercept = userRule.shouldInterceptHttpsReq(req);
console.log(color.green("\nreceived https CONNECT request " + host));
if(shouldIntercept){
console.log("==>will forward to local https server");
}else{
//TODO : remote port other than 433
console.log("==>meet the rules, will forward to local https server");
//forward the https-request to local https server
httpsServerMgrInstance.fetchPort(hostname,userRequestHandler,function(err,port){
if(!err && port){
try{
var conn = net.connect(port, 'localhost', function(){ //TODO : localhost -> server
socket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', function() {
conn.pipe(socket);
socket.pipe(conn);
});
});
conn.on("error",function(e){
console.log("err when connect to __host".replace(/__host/,hostname));
});
}catch(e){
console.log("err when connect to local https server (__hostname)".replace(/__hostname/,hostname));//TODO
}
}else{
console.log("err fetch HTTPS server for host:" + hostname);
}
});
console.log("==>will bypass the man-in-the-middle proxy");
}
//record
resourceInfo = {
host : host,
method : req.method,
path : "",
url : "https://" + host,
req : req,
startTime : new Date().getTime()
};
resourceInfoId = GLOBAL.recorder.appendRecord(resourceInfo);
var proxyPort, proxyHost;
async.series([
//find port
function(callback){
if(shouldIntercept){
//TODO : remote port other than 433
httpsServerMgrInstance.fetchPort(host,userRequestHandler,function(err,port){
if(!err && port){
proxyPort = port;
proxyHost = "127.0.0.1";
callback();
}else{
callback(err);
}
});
}else{
proxyPort = targetPort;
proxyHost = host;
callback();
}
//connect
},function(callback){
try{
var conn = net.connect(proxyPort, proxyHost, function(){
socket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', function() {
conn.pipe(socket);
socket.pipe(conn);
callback();
});
});
conn.on("error",function(e){
console.log("err when connect to __host".replace(/__host/,host));
});
}catch(e){
console.log("err when connect to remote https server (__host)".replace(/__host/,host));
}
//update record
},function(callback){
resourceInfo.endTime = new Date().getTime();
resourceInfo.statusCode = "200";
resourceInfo.resHeader = {};
resourceInfo.resBody = "";
resourceInfo.length = 0;
try{
GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo);
}catch(e){}
callback();
}
],function(err,result){
if(err){
console.log("err " + err);
throw err;
}
});
}
// {"Content-Encoding":"gzip"} --> {"content-encoding":"gzip"}
function lower_keys(obj){
for(var key in obj){
var val = obj[key];
delete obj[key];
obj[key.toLowerCase()] = val;
}
return obj;
}
//TODO : reactive this function
function setRules(newRule){
if(!newRule){
return;
}
if(!newRule.map || !newRule.httpsConfig){
throw(new Error("invalid rule schema"));
}else{
handleRule = newRule;
userRule = newRule;
}
}

148
lib/rule.js
View File

@@ -1,148 +0,0 @@
module.exports = {
/*
thess functions are required
you may leave their bodies blank if necessary
*/
//whether to intercept this request by local logic
//if the return value is true, anyproxy will call dealLocalResponse to get response data and will not send request to remote server anymore
shouldUseLocalResponse : function(req){
if(req.method == "OPTIONS"){
return true;
}else{
return false;
}
},
//response to user via local logic, be called when shouldUseLocalResponse returns true
//you should call callback(statusCode,resHeader,responseData)
//e.g. callback(200,{"content-type":"text/html"},"hello world")
dealLocalResponse : function(req,callback){
if(req.method == "OPTIONS"){
callback(200,mergeCORSHeader(req.headers),"");
}
},
//req is user's request sent to the proxy server
// option is how the proxy server will send request to the real server. i.e. require("http").request(option,function(){...})
//you may return a customized option to replace the original option
replaceRequestOption : function(req,option){
var newOption = option;
// newOption = {
// hostname : "www.example.com",
// port : "80",
// path : '/',
// method : "GET",
// headers : {}
// };
return newOption;
},
//replace the request protocol when sending to the real server
//protocol : "http" or "https"
replaceRequestProtocol:function(req,protocol){
var newProtocol = protocol;
return newProtocol;
},
//replace the statusCode before it's sent to the user
replaceResponseStatusCode: function(req,res,statusCode){
var newStatusCode = statusCode;
return newStatusCode;
},
//replace the httpHeader before it's sent to the user
replaceResponseHeader: function(req,res,header){
var newHeader = header;
newHeader = mergeCORSHeader(req.headers, newHeader);
newHeader = disableCacheHeader(newHeader);
return newHeader;
},
//replace the response from the server before it's sent to the user
//you may return either a Buffer or a string
//serverResData is a Buffer, you may get its content by calling serverResData.toString()
replaceServerResData: function(req,res,serverResData){
if(/html/i.test(res.headers['content-type'])){
var newDataStr = serverResData.toString(); //TODO : failed to decode data
// newDataStr += "hello world!";
return newDataStr;
}else{
return serverResData;
}
},
//add a pause before sending response to user
pauseBeforeSendingResponse : function(req,res){
var timeInMS = 100; //delay all requests for 0.1s
return timeInMS;
}
};
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS
function mergeCORSHeader(reqHeader,originHeader){
var targetObj = originHeader || {};
delete targetObj["Access-Control-Allow-Credentials"];
delete targetObj["Access-Control-Allow-Origin"];
delete targetObj["Access-Control-Allow-Methods"];
delete targetObj["Access-Control-Allow-Headers"];
targetObj["access-control-allow-credentials"] = "true";
targetObj["access-control-allow-origin"] = reqHeader['origin'] || "-___-||";
targetObj["access-control-allow-methods"] = "GET, POST, PUT";
targetObj["access-control-allow-headers"] = reqHeader['access-control-request-headers'] || "-___-||";
return targetObj;
}
function disableCacheHeader(header){
header = header || {};
header["Cache-Control"] = "no-cache, no-store, must-revalidate";
header["Pragma"] = "no-cache";
header["Expires"] = 0;
header["server"] = "anyproxy server";
header["x-powered-by"] = "Anyproxy";
return header;
}
//try to mactch rule file
// for(var index in handleRule.map){
// var rule = handleRule.map[index];
// var hostTest = new RegExp(rule.host).test(host),
// pathTest = new RegExp(rule.path).test(path);
// if(hostTest && pathTest && (rule.localFile || rule.localDir) ){
// console.log("==>meet the rules, will map to local file");
// var targetLocalfile = rule.localFile;
// //localfile not set, map to dir
// if(!targetLocalfile){ //find file in dir, /a/b/file.html -> dir + b/file.html
// var remotePathWithoutPrefix = path.replace(new RegExp(rule.path),""); //remove prefix
// targetLocalfile = pathUtil.join(rule.localDir,remotePathWithoutPrefix);
// }
// console.log("==>local file: " + targetLocalfile);
// if(fs.existsSync(targetLocalfile)){
// try{
// var fsStream = fs.createReadStream(targetLocalfile);
// userRes.writeHead(200,mergeCORSHeader( req.headers,{}) ); //CORS for localfiles
// fsStream.pipe(userRes);
// ifLocalruleMatched = true;
// break;
// }catch(e){
// console.log(e.message);
// }
// }else{
// console.log("file not exist : " + targetLocalfile);
// }
// }
// }

28
lib/rule_default.js Normal file
View File

@@ -0,0 +1,28 @@
module.exports = {
shouldUseLocalResponse : function(req){
},
dealLocalResponse : function(req,callback){
},
replaceRequestOption : function(req,option){
},
replaceRequestProtocol:function(req,protocol){
},
replaceResponseStatusCode: function(req,res,statusCode){
},
replaceResponseHeader: function(req,res,header){
},
replaceServerResData: function(req,res,serverResData){
},
pauseBeforeSendingResponse : function(req,res){
},
shouldInterceptHttpsReq :function(req){
}
};