diff --git a/lib/recorder.js b/lib/recorder.js index 504bc60..5f65a45 100644 --- a/lib/recorder.js +++ b/lib/recorder.js @@ -45,16 +45,8 @@ function Recorder(){ if(!id || !info.resBody) return; //add to body map //do not save image data - if(/image/.test(info.res.headers['content-type'])){ + if(/image/.test(info.resHeader['content-type'])){ self.recordBodyMap[id] = "(image)"; - }else if(/gzip/.test(info.res.headers['content-encoding'])){ - zlib.unzip(info.resBody,function(err,buffer){ - if(err){ - self.recordBodyMap[id] = "(err when unzip response buffer)"; - }else{ - self.recordBodyMap[id] = buffer.toString(); - } - }); }else{ self.recordBodyMap[id] = info.resBody.toString(); } @@ -81,7 +73,7 @@ function normalizeInfo(id,info){ //general singleRecord._id = id; - singleRecord.id = id; + singleRecord.id = id; singleRecord.url = info.url; singleRecord.host = info.host; singleRecord.path = info.path; @@ -92,13 +84,13 @@ function normalizeInfo(id,info){ singleRecord.startTime = info.startTime; //res - if(info.res){ - singleRecord.statusCode= info.res.statusCode; + if(info.endTime){ + singleRecord.statusCode= info.statusCode; singleRecord.endTime = info.endTime; - singleRecord.resHeader = info.res.headers; + singleRecord.resHeader = info.resHeader; singleRecord.length = info.length; - if(info.res.headers['content-type']){ - singleRecord.mime = info.res.headers['content-type'].split(";")[0]; + if(info.resHeader['content-type']){ + singleRecord.mime = info.resHeader['content-type'].split(";")[0]; }else{ singleRecord.mime = ""; } diff --git a/lib/requestHandler.js b/lib/requestHandler.js index 33a1213..16a2d4d 100644 --- a/lib/requestHandler.js +++ b/lib/requestHandler.js @@ -8,43 +8,28 @@ var http = require("http"), async = require('async'), color = require("colorful"), Buffer = require('buffer').Buffer, - httpsServerMgr = require("./httpsServerMgr"), - userRule = require("./rule.js"); //TODO - to be configurable + httpsServerMgr = require("./httpsServerMgr"); -var httpsServerMgrInstance = new httpsServerMgr(); - -//default rule -var handleRule = { - map :[ - // { - // host :".", - // path :"/path/test", - // localFile :"", - // localDir :"~/" - // } - ] - ,httpsConfig:{ - bypassAll : true, - interceptDomains:["^.*alibaba-inc\.com$"] - } -}; +var httpsServerMgrInstance = new httpsServerMgr(), + userRule = require("./rule_default.js"); //default rule file function userRequestHandler(req,userRes){ - var host = req.headers.host, + var host = req.headers.host, urlPattern = url.parse(req.url), path = urlPattern.path, - callback = null, //TODO : remove callback protocol = (!!req.connection.encrypted && !/http:/.test(req.url)) ? "https" : "http", - resourceInfo = {}, + resourceInfo, resourceInfoId = -1; //record - resourceInfo.host = host; - resourceInfo.method = req.method; - resourceInfo.path = path; - resourceInfo.url = protocol + "://" + host + path; - resourceInfo.req = req; - resourceInfo.startTime = new Date().getTime(); + resourceInfo = { + host : host, + method : req.method, + path : path, + url : protocol + "://" + host + path, + req : req, + startTime : new Date().getTime() + }; try{ resourceInfoId = GLOBAL.recorder.appendRecord(resourceInfo); @@ -101,14 +86,17 @@ function userRequestHandler(req,userRes){ var statusCode = res.statusCode; statusCode = userRule.replaceResponseStatusCode(req,res,statusCode) || statusCode; - var resHeader = res.headers; - resHeader = userRule.replaceResponseHeader(req,res,resHeader) || resHeader; + var resHeader = userRule.replaceResponseHeader(req,res,res.headers) || res.headers; + resHeader = lower_keys(resHeader); - //remove content-encoding - // delete resHeader['content-encoding']; + /* remove gzip related header, and ungzip the content */ + var ifServerGzipped = /gzip/i.test(resHeader['content-encoding']); + delete resHeader['content-encoding']; + delete resHeader['content-length']; userRes.writeHead(statusCode, resHeader); + //waiting for data var resData = [], length; @@ -122,12 +110,11 @@ function userRequestHandler(req,userRes){ userCustomResData; async.series([ - //TODO : manage gzip - //unzip server res + //ungzip server res function(callback){ serverResData = Buffer.concat(resData); - if(/gzip/i.test(res.headers['content-encoding'])){ + if(ifServerGzipped ){ zlib.gunzip(serverResData,function(err,buff){ serverResData = buff; callback(); @@ -140,20 +127,6 @@ function userRequestHandler(req,userRes){ },function(callback){ userCustomResData = userRule.replaceServerResData(req,res,serverResData); - - //gzip users' string if necessary - if(typeof userCustomResData == "string" && /gzip/i.test(res.headers['content-encoding']) ){ - zlib.gzip(userCustomResData,function(err,data){ - userCustomResData = data; - console.log(data); - callback(); - }); - }else{ - callback(); - } - - //generate response data - },function(callback){ serverResData = userCustomResData || serverResData; callback(); @@ -168,18 +141,17 @@ function userRequestHandler(req,userRes){ //send response },function(callback){ - userRes.write(serverResData); - userRes.end(); - + userRes.end(serverResData); callback(); //udpate record info },function(callback){ - resourceInfo.endTime = new Date().getTime(); - resourceInfo.res = res; //TODO : replace res header / statusCode ? - resourceInfo.resBody = serverResData; - resourceInfo.length = serverResData.length; - + resourceInfo.endTime = new Date().getTime(); + resourceInfo.statusCode = statusCode; + resourceInfo.resHeader = resHeader; + resourceInfo.resBody = serverResData; + resourceInfo.length = serverResData.length; + try{ GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo); }catch(e){} @@ -208,80 +180,112 @@ function userRequestHandler(req,userRes){ } function connectReqHandler(req, socket, head){ - var hostname = req.url.split(":")[0], + var host = req.url.split(":")[0], targetPort= req.url.split(":")[1], - httpsRule = handleRule.httpsConfig; + resourceInfo, + resourceInfoId; - var shouldBypass = !!httpsRule.bypassAll; - if(!shouldBypass){ //read rules - shouldBypass = true; - for(var index in httpsRule.interceptDomains){ - var reg = new RegExp(httpsRule.interceptDomains[index]); - if( reg.test(hostname) ){ - shouldBypass = false; - break; - } - } - } - - console.log(color.green("\nreceived https CONNECT request " + hostname)); - - if(shouldBypass){ - console.log("==>will bypass the man-in-the-middle proxy"); - try{ - var conn = net.connect(targetPort, hostname, function(){ - socket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', function() { - conn.pipe(socket); - socket.pipe(conn); - }); - }); - - conn.on("error",function(e){ - console.log("err when connect to __host".replace(/__host/,hostname)); - }); - }catch(e){ - console.log("err when connect to remote https server (__hostname)".replace(/__hostname/,hostname));//TODO - } + var shouldIntercept = userRule.shouldInterceptHttpsReq(req); + console.log(color.green("\nreceived https CONNECT request " + host)); + if(shouldIntercept){ + console.log("==>will forward to local https server"); }else{ - //TODO : remote port other than 433 - console.log("==>meet the rules, will forward to local https server"); - - //forward the https-request to local https server - httpsServerMgrInstance.fetchPort(hostname,userRequestHandler,function(err,port){ - if(!err && port){ - try{ - var conn = net.connect(port, 'localhost', function(){ //TODO : localhost -> server - socket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', function() { - conn.pipe(socket); - socket.pipe(conn); - }); - }); - - conn.on("error",function(e){ - console.log("err when connect to __host".replace(/__host/,hostname)); - }); - }catch(e){ - console.log("err when connect to local https server (__hostname)".replace(/__hostname/,hostname));//TODO - } - - }else{ - console.log("err fetch HTTPS server for host:" + hostname); - } - }); + console.log("==>will bypass the man-in-the-middle proxy"); } + + //record + resourceInfo = { + host : host, + method : req.method, + path : "", + url : "https://" + host, + req : req, + startTime : new Date().getTime() + }; + resourceInfoId = GLOBAL.recorder.appendRecord(resourceInfo); + + var proxyPort, proxyHost; + async.series([ + + //find port + function(callback){ + if(shouldIntercept){ + //TODO : remote port other than 433 + httpsServerMgrInstance.fetchPort(host,userRequestHandler,function(err,port){ + if(!err && port){ + proxyPort = port; + proxyHost = "127.0.0.1"; + callback(); + }else{ + callback(err); + } + }); + + }else{ + proxyPort = targetPort; + proxyHost = host; + + callback(); + } + + //connect + },function(callback){ + try{ + var conn = net.connect(proxyPort, proxyHost, function(){ + socket.write('HTTP/' + req.httpVersion + ' 200 OK\r\n\r\n', 'UTF-8', function() { + conn.pipe(socket); + socket.pipe(conn); + callback(); + }); + }); + + conn.on("error",function(e){ + console.log("err when connect to __host".replace(/__host/,host)); + }); + }catch(e){ + console.log("err when connect to remote https server (__host)".replace(/__host/,host)); + } + + //update record + },function(callback){ + resourceInfo.endTime = new Date().getTime(); + resourceInfo.statusCode = "200"; + resourceInfo.resHeader = {}; + resourceInfo.resBody = ""; + resourceInfo.length = 0; + + try{ + GLOBAL.recorder.updateRecord(resourceInfoId,resourceInfo); + }catch(e){} + + callback(); + } + ],function(err,result){ + if(err){ + console.log("err " + err); + throw err; + } + }); +} + +// {"Content-Encoding":"gzip"} --> {"content-encoding":"gzip"} +function lower_keys(obj){ + for(var key in obj){ + var val = obj[key]; + delete obj[key]; + + obj[key.toLowerCase()] = val; + } + + return obj; } -//TODO : reactive this function function setRules(newRule){ if(!newRule){ return; - } - - if(!newRule.map || !newRule.httpsConfig){ - throw(new Error("invalid rule schema")); }else{ - handleRule = newRule; + userRule = newRule; } } diff --git a/lib/rule.js b/lib/rule.js deleted file mode 100644 index 45edebd..0000000 --- a/lib/rule.js +++ /dev/null @@ -1,148 +0,0 @@ -module.exports = { - /* - thess functions are required - you may leave their bodies blank if necessary - */ - - //whether to intercept this request by local logic - //if the return value is true, anyproxy will call dealLocalResponse to get response data and will not send request to remote server anymore - shouldUseLocalResponse : function(req){ - if(req.method == "OPTIONS"){ - return true; - }else{ - return false; - } - }, - - //response to user via local logic, be called when shouldUseLocalResponse returns true - //you should call callback(statusCode,resHeader,responseData) - //e.g. callback(200,{"content-type":"text/html"},"hello world") - dealLocalResponse : function(req,callback){ - if(req.method == "OPTIONS"){ - callback(200,mergeCORSHeader(req.headers),""); - } - }, - - //req is user's request sent to the proxy server - // option is how the proxy server will send request to the real server. i.e. require("http").request(option,function(){...}) - //you may return a customized option to replace the original option - replaceRequestOption : function(req,option){ - var newOption = option; - - // newOption = { - // hostname : "www.example.com", - // port : "80", - // path : '/', - // method : "GET", - // headers : {} - // }; - return newOption; - }, - - //replace the request protocol when sending to the real server - //protocol : "http" or "https" - replaceRequestProtocol:function(req,protocol){ - var newProtocol = protocol; - return newProtocol; - }, - - //replace the statusCode before it's sent to the user - replaceResponseStatusCode: function(req,res,statusCode){ - var newStatusCode = statusCode; - return newStatusCode; - }, - - //replace the httpHeader before it's sent to the user - replaceResponseHeader: function(req,res,header){ - var newHeader = header; - - newHeader = mergeCORSHeader(req.headers, newHeader); - newHeader = disableCacheHeader(newHeader); - return newHeader; - }, - - //replace the response from the server before it's sent to the user - //you may return either a Buffer or a string - //serverResData is a Buffer, you may get its content by calling serverResData.toString() - replaceServerResData: function(req,res,serverResData){ - if(/html/i.test(res.headers['content-type'])){ - var newDataStr = serverResData.toString(); //TODO : failed to decode data - // newDataStr += "hello world!"; - return newDataStr; - }else{ - return serverResData; - } - }, - - //add a pause before sending response to user - pauseBeforeSendingResponse : function(req,res){ - var timeInMS = 100; //delay all requests for 0.1s - return timeInMS; - } - -}; - -// https://developer.mozilla.org/en-US/docs/Web/HTTP/Access_control_CORS -function mergeCORSHeader(reqHeader,originHeader){ - var targetObj = originHeader || {}; - - delete targetObj["Access-Control-Allow-Credentials"]; - delete targetObj["Access-Control-Allow-Origin"]; - delete targetObj["Access-Control-Allow-Methods"]; - delete targetObj["Access-Control-Allow-Headers"]; - - targetObj["access-control-allow-credentials"] = "true"; - targetObj["access-control-allow-origin"] = reqHeader['origin'] || "-___-||"; - targetObj["access-control-allow-methods"] = "GET, POST, PUT"; - targetObj["access-control-allow-headers"] = reqHeader['access-control-request-headers'] || "-___-||"; - - return targetObj; -} - - -function disableCacheHeader(header){ - header = header || {}; - header["Cache-Control"] = "no-cache, no-store, must-revalidate"; - header["Pragma"] = "no-cache"; - header["Expires"] = 0; - header["server"] = "anyproxy server"; - header["x-powered-by"] = "Anyproxy"; - - return header; -} - -//try to mactch rule file -// for(var index in handleRule.map){ -// var rule = handleRule.map[index]; - - -// var hostTest = new RegExp(rule.host).test(host), -// pathTest = new RegExp(rule.path).test(path); - -// if(hostTest && pathTest && (rule.localFile || rule.localDir) ){ -// console.log("==>meet the rules, will map to local file"); - -// var targetLocalfile = rule.localFile; - -// //localfile not set, map to dir -// if(!targetLocalfile){ //find file in dir, /a/b/file.html -> dir + b/file.html -// var remotePathWithoutPrefix = path.replace(new RegExp(rule.path),""); //remove prefix -// targetLocalfile = pathUtil.join(rule.localDir,remotePathWithoutPrefix); -// } - -// console.log("==>local file: " + targetLocalfile); -// if(fs.existsSync(targetLocalfile)){ -// try{ -// var fsStream = fs.createReadStream(targetLocalfile); -// userRes.writeHead(200,mergeCORSHeader( req.headers,{}) ); //CORS for localfiles -// fsStream.pipe(userRes); -// ifLocalruleMatched = true; -// break; -// }catch(e){ -// console.log(e.message); -// } -// }else{ -// console.log("file not exist : " + targetLocalfile); -// } -// } -// } \ No newline at end of file diff --git a/lib/rule_default.js b/lib/rule_default.js new file mode 100644 index 0000000..b80d8c6 --- /dev/null +++ b/lib/rule_default.js @@ -0,0 +1,28 @@ +module.exports = { + shouldUseLocalResponse : function(req){ + }, + + dealLocalResponse : function(req,callback){ + }, + + replaceRequestOption : function(req,option){ + }, + + replaceRequestProtocol:function(req,protocol){ + }, + + replaceResponseStatusCode: function(req,res,statusCode){ + }, + + replaceResponseHeader: function(req,res,header){ + }, + + replaceServerResData: function(req,res,serverResData){ + }, + + pauseBeforeSendingResponse : function(req,res){ + }, + + shouldInterceptHttpsReq :function(req){ + } +}; \ No newline at end of file diff --git a/postReceiver.js b/postReceiver.js deleted file mode 100644 index d65e25a..0000000 --- a/postReceiver.js +++ /dev/null @@ -1,18 +0,0 @@ -var http= require("http"); - -var s = http.createServer(function(req,res) { - var total = ""; - req.on("data",function(chunk){ - total += chunk; - }); - - req.on("end",function(){ - console.log(total); - }); - - console.log(req); - // body... -}); - -s.listen(80); - diff --git a/proxy.js b/proxy.js index 8775137..7e95430 100644 --- a/proxy.js +++ b/proxy.js @@ -40,7 +40,7 @@ function proxyServer(type, port, hostname,ruleFile){ if(ruleFile){ if(fs.existsSync(ruleFile)){ try{ //for abs path - requestHandler.setRules(require(ruleFile)); //todo : require path + requestHandler.setRules(require(ruleFile)); }catch(e){ //for relative path requestHandler.setRules(require("./" + ruleFile)); } @@ -73,9 +73,10 @@ function proxyServer(type, port, hostname,ruleFile){ } }, - //listen CONNECT method for https over http function(callback){ + //listen CONNECT method for https over http self.httpProxyServer.on('connect',requestHandler.connectReqHandler); + self.httpProxyServer.listen(proxyPort); callback(null); } @@ -135,9 +136,11 @@ function startWebServer(port){ console.log(color.green(tipText)); - //web socket interface var wss = new WebSocketServer({port: DEFAULT_WEBSOCKET_PORT}); + wss.on("connection",function(ws){ + console.log("wss connection"); + }); wss.broadcast = function(data) { for(var i in this.clients){ this.clients[i].send(data); diff --git a/rule_sample.js b/rule_sample.js deleted file mode 100644 index b585c3b..0000000 --- a/rule_sample.js +++ /dev/null @@ -1,39 +0,0 @@ -var rules = { - "map" :[ - { - "host" :/./, //regExp - "path" :/\/path\/test/, //regExp - "localFile" :"", //this file will be returned to user when host and path pattern both meets the request - "localDir" :"~/" //find the file of same name in localdir. anyproxy will not read localDir settings unless localFile is falsy - } - // ,{ - // "host" :/./, - // "path" :/\.(png|gif|jpg|jpeg)/, - // "localFile" :"/Users/Stella/tmp/test.png", - // "localDir" :"~/" - // } - ,{ - "host" :/./, - "path" :/tps/, - "localFile" :"", - "localDir" :"/Users/Stella/tmp/" - },{ - "host" :/./, - "path" :/response\.(json)/ - },{ - "host" :/./, - "path" :/html/, - "callback" :function(res){ - //remoty.js will be inject into response via callback - res.write("