move response data from memory to local cache file

lib/recorder.js

@@ -2,15 +2,18 @@
 var zlib      = require('zlib'),
     Datastore = require('nedb'), 
     util      = require("util"),
+    path      = require("path"),
     fs        = require("fs"),
     events    = require('events'),
     iconv     = require('iconv-lite'),
+    proxyUtil = require("./util"),
     logUtil   = require("./log");
 
 //option.filename
 function Recorder(option){
     var self = this,
-        id   = 1,
+        id        = 1,
+        cachePath = proxyUtil.generateCacheDir(),
         db;
 
     option = option || {};
@@ -87,31 +90,40 @@ function Recorder(option){
     };
 
     //update recordBody if exits
+
+    //TODO : trigger update callback
+    var BODY_FILE_PRFIX = "res_body_";
     self.updateRecordBody =function(id,info){
         if(id == -1) return;
 
         if(!id || !info.resBody) return;
         //add to body map
         //ignore image data
-        if(/image/.test(info.resHeader['content-type'])){
+        var bodyFile = path.join(cachePath,BODY_FILE_PRFIX + id);
-            self.recordBodyMap[id] = "(image)";
+        fs.writeFile(bodyFile, info.resBody);
-        }else{
-            self.recordBodyMap[id] = info.resBody;
-        }
     };
 
-    self.getBody = function(id){
+    self.getBody = function(id,cb){
         if(id < 0){
-            return "";
+            cb && cb("");
         }
 
-        return self.recordBodyMap[id] || "";
+        var bodyFile = path.join(cachePath,BODY_FILE_PRFIX + id);
+        fs.access(bodyFile, fs.F_OK | fs.R_OK ,function(err){
+            if(err){
+                cb && cb(err);
+            }else{
+                fs.readFile(bodyFile,cb);
+            }
+        });
     };
 
-    self.getBodyUTF8 = function(id,cb){
+    self.getDecodedBody = function(id,cb){
-        var bodyContent = self.getBody(id),
+        var result = {
-            result      = "";
+            type    : "unknown",
-
+            mime    : "",
+            content : ""
+        };
         GLOBAL.recorder.getSingleRecord(id,function(err,doc){
             //check whether this record exists
             if(!doc || !doc[0]){
@@ -119,24 +131,39 @@ function Recorder(option){
                 return;
             }
 
-            if(!bodyContent){
+            self.getBody(id,function(err,bodyContent){
-                cb(null,result);
+                if(err){
-            }else{
+                    cb(err);
-                var record      = doc[0],
+                }else if(!bodyContent){
-                    resHeader   = record['resHeader'] || {};
+                    cb(null,result);
-                try{
+                }else{
-                    var charsetMatch = JSON.stringify(resHeader).match(/charset="?([a-zA-Z0-9\-]+)"?/);
+                    var record      = doc[0],
-                    if(charsetMatch && charsetMatch.length > 1){
+                        resHeader   = record['resHeader'] || {};
-                        var currentCharset = charsetMatch[1].toLowerCase();
+                    try{
-                        if(currentCharset != "utf-8" && iconv.encodingExists(currentCharset)){
+                        var headerStr    = JSON.stringify(resHeader),
-                            bodyContent = iconv.decode(bodyContent, currentCharset);
+                            charsetMatch = headerStr.match(/charset="?([a-zA-Z0-9\-]+)"?/),
+                            imageMatch   = resHeader && resHeader["content-type"];
+
+                        if(charsetMatch && charsetMatch.length){
+
+                            var currentCharset = charsetMatch[1].toLowerCase();
+                            if(currentCharset != "utf-8" && iconv.encodingExists(currentCharset)){
+                                bodyContent = iconv.decode(bodyContent, currentCharset);
+                            }
+                            result.type    = "text";
+                            result.content = bodyContent.toString();
+                        }else if(imageMatch){
+
+                            result.type    = "image";
+                            result.mime    = imageMatch;
+                            result.content = bodyContent;
                         }
-                    }
+                    }catch(e){}
-                }catch(e){}
 
-                cb(null,bodyContent.toString());
+                    cb(null,result);
-            }
+                }
 
+            });
         });
     };
 

lib/requestHandler.js

@@ -24,6 +24,7 @@ function userRequestHandler(req,userRes){
         in http  server : http://www.example.com/a/b/c
         in https server : /a/b/c
     */
+
     var host               = req.headers.host,
         protocol           = (!!req.connection.encrypted && !/^http:/.test(req.url)) ? "https" : "http",
         fullUrl            = protocol === "http" ? req.url : (protocol + '://' + host + req.url),
@@ -33,6 +34,9 @@ function userRequestHandler(req,userRes){
         resourceInfoId     = -1,
         reqData;
 
+    // console.log(req.url);
+    // console.log(path);
+
     //record
     resourceInfo = {
         host      : host,

lib/util.js

@@ -1,5 +1,7 @@
 var fs   = require("fs"),
-    path = require("path");
+    path = require("path"),
+    exec = require('child_process').exec;
+
 
 // {"Content-Encoding":"gzip"} --> {"content-encoding":"gzip"}
 module.exports.lower_keys = function(obj){
@@ -41,6 +43,27 @@ module.exports.getAnyProxyHome = function(){
     return home;
 }
 
+var CACHE_DIR_PREFIX = "cache_r";
+module.exports.generateCacheDir = function(){
+    var rand      = Math.floor(Math.random() * 1000000),
+        cachePath = path.join(util.getAnyProxyHome(),"./" + CACHE_DIR_PREFIX + rand);
+
+    fs.mkdirSync(cachePath,0777);
+    return cachePath;
+}
+
+module.exports.clearCacheDir = function(cb){
+    var home  = util.getAnyProxyHome(),
+        isWin = /^win/.test(process.platform);
+
+    var dirNameWildCard = CACHE_DIR_PREFIX + "*";
+    if(isWin){
+        exec("for /D %f in (" + dirNameWildCard + ") do rmdir %f /s /q",{cwd : home},cb);
+    }else{
+        exec("rm -rf " + dirNameWildCard + "",{cwd : home},cb);        
+    }
+}
+
 module.exports.simpleRender = function(str, object, regexp){
     return String(str).replace(regexp || (/\{\{([^{}]+)\}\}/g), function(match, name){
         if (match.charAt(0) == '\\') return match.slice(1);

lib/webInterface.js

@@ -48,6 +48,36 @@ function webInterface(config){
         });
     });
 
+    app.get("/fetchBody",function(req,res){
+        var query = req.query;
+        if(query && query.id){
+            GLOBAL.recorder.getDecodedBody(query.id, function(err, result){
+                if(err || !result || !result.content){
+                    res.json({});
+                }else if(result.type && result.type == "image" && result.mime){
+                    if(query.raw){
+                        //TODO : cache query result
+                        res.type(result.mime).end(result.content);
+                    }else{
+                        res.json({
+                            id      : query.id,
+                            type    : result.type,
+                            ref     : "/fetchBody?id=" + query.id + "&raw=true"
+                        });
+                    }
+                }else{
+                    res.json({
+                        id      : query.id,
+                        type    : result.type,
+                        content : result.content
+                    });
+                }
+            });
+        }else{
+            res.end({});
+        }
+    });
+
     app.get("/fetchCrtFile",function(req,res){
         if(crtFilePath){
             res.setHeader("Content-Type","application/x-x509-ca-cert");

lib/wsServer.js

@@ -23,8 +23,8 @@ function resToMsg(msg,cb){
     }
 
     if(jsonData.type == "reqBody" && jsonData.id){
-        result.type ="body";
+        result.type = "body";
-        GLOBAL.recorder.getBodyUTF8(jsonData.id, function(err, data){
+        GLOBAL.recorder.getBody(jsonData.id, function(err, data){
             if(err){
                 result.content = {
                     id    : null,
@@ -34,7 +34,7 @@ function resToMsg(msg,cb){
             }else{
                 result.content = {
                     id   : jsonData.id,
-                    body : data
+                    body : data.toString()
                 };
             }
             cb && cb(result);

package.json

@@ -25,7 +25,7 @@
     "ws": "^0.4.32"
   },
   "devDependencies": {
-    "proxy-eval": ">=1.1.1"
+    "proxy-eval": ">=1.1.2"
   },
   "scripts": {
     "test": "sh test.sh"

proxy.js

@@ -69,11 +69,7 @@ function proxyServer(option){
         logUtil.setPrintStatus(false);
     }
 
-    if(option.dbFile){
+
-        GLOBAL.recorder = new Recorder({filename: option.dbFile});
-    }else{
-        GLOBAL.recorder = new Recorder();
-    }
 
     if(!!option.interceptHttps){
         default_rule.setInterceptFlag(true);
@@ -95,6 +91,18 @@ function proxyServer(option){
 
     async.series(
         [
+            //clear cache dir, prepare recorder
+            function(callback){
+                util.clearCacheDir(function(){
+                    if(option.dbFile){
+                        GLOBAL.recorder = new Recorder({filename: option.dbFile});
+                    }else{
+                        GLOBAL.recorder = new Recorder();
+                    }
+                    callback();
+                });
+            },
+
             //creat proxy server
             function(callback){
                 if(proxyType == T_TYPE_HTTPS){

test/large_post.js

@@ -0,0 +1,37 @@
+var proxy       = require("../proxy.js"),
+	proxyTester = require("proxy-eval"),
+	WebSocket   = require("ws"),
+    Buffer      = require("buffer").Buffer,
+    express     = require("express");
+
+var app = express()
+ 
+app.post('/', function (req, res) {
+    var bigBody = new Buffer(1024 * 1024 * 10);
+    res.send( bigBody ); //10 mb
+});
+app.listen(3000);
+
+function test(){
+    //test the basic availibility of proxy server
+    setTimeout(function(){
+        var testParam = {
+            proxy         : 'http://127.0.0.1:8001/',
+            reqTimeout    : 4500,
+            httpGetUrl    : "",
+            httpPostUrl   : "http://127.0.0.1:3000/",
+            httpPostBody  : "123",
+            httpsGetUrl   : "",
+            httpsPostUrl  : "",
+            httpsPostBody : ""
+        };
+    	proxyTester.test(testParam ,function(results){
+            process.exit();
+        });
+    },1000);
+};
+
+setTimeout(function(){
+    test();
+},3000);
+

test/test.js

@@ -44,7 +44,13 @@ exports.avalibility = function(test){
 
     //test the basic availibility of proxy server
     setTimeout(function(){
-    	proxyTester.test({proxy : 'http://127.0.0.1:8995',reqTimeout:4500} ,function(results){
+        var testParam = {
+            proxy         : 'http://127.0.0.1:8995',
+            reqTimeout    : 4500,
+            httpsPostUrl  : "http://www.sample.com/"
+            httpsPostBody : "123"
+        };
+    	proxyTester.test(testParam ,function(results){
     		var successCount = 0;
     		results.map(function(item){
     			item.success && ++successCount;

web/build/detailPanel.js

@@ -12,11 +12,23 @@ function init(React){
 			    id   = self.props.data.id;
 			if(!id) return;
 
-			ws.reqBody(id,function(content){
+			jQuery.get("/fetchBody?id=" + id ,function(resObj){
-				if(content.id == self.props.data.id){
+				if(resObj && resObj.id){
-					self.setState({
+					if(resObj.type && resObj.type == "image" && resObj.ref){
-						body : content
+						self.setState({
-					});
+							body : {
+								img : resObj.ref,
+								id  : resObj.id
+							}
+						});
+					}else if(resObj.content){
+						self.setState({
+							body : {
+								body : resObj.content,
+								id   : resObj.id
+							}
+						});
+					}
 				}
 			});
 		},
@@ -55,9 +67,13 @@ function init(React){
 			);
 
 			if(this.props.data.statusCode){
-
 				if(this.state.body.id == this.props.data.id){
-					bodyContent = (React.createElement("pre", {className: "resBodyContent"}, this.state.body.body));
+					if(this.state.body.img){
+						var imgEl = { __html : '<img src="'+ this.state.body.img +'" />'};
+						bodyContent = (React.createElement("div", {dangerouslySetInnerHTML: imgEl}));
+					}else{
+						bodyContent = (React.createElement("pre", {className: "resBodyContent"}, this.state.body.body));
+					}
 				}else{
 					bodyContent = null;
 					this.loadBody();

web/css/page.css

@@ -186,6 +186,11 @@ body, html {
 	word-wrap:break-word;
 }
 
+.resBody .resBodyContent img{
+	max-width: 500px;
+	max-height: 500px;
+}
+
 .subTitle{
 	padding-left: 6px;
 	border-left: 3px solid #1FA2D6;

web/page.js

@@ -20342,11 +20342,23 @@
 				    id   = self.props.data.id;
 				if(!id) return;
 
-				ws.reqBody(id,function(content){
+				jQuery.get("/fetchBody?id=" + id ,function(resObj){
-					if(content.id == self.props.data.id){
+					if(resObj && resObj.id){
-						self.setState({
+						if(resObj.type && resObj.type == "image" && resObj.ref){
-							body : content
+							self.setState({
-						});
+								body : {
+									img : resObj.ref,
+									id  : resObj.id
+								}
+							});
+						}else if(resObj.content){
+							self.setState({
+								body : {
+									body : resObj.content,
+									id   : resObj.id
+								}
+							});
+						}
 					}
 				});
 			},
@@ -20385,9 +20397,13 @@
 				);
 
 				if(this.props.data.statusCode){
-
 					if(this.state.body.id == this.props.data.id){
-						bodyContent = (React.createElement("pre", {className: "resBodyContent"}, this.state.body.body));
+						if(this.state.body.img){
+							var imgEl = { __html : '<img src="'+ this.state.body.img +'" />'};
+							bodyContent = (React.createElement("div", {dangerouslySetInnerHTML: imgEl}));
+						}else{
+							bodyContent = (React.createElement("pre", {className: "resBodyContent"}, this.state.body.body));
+						}
 					}else{
 						bodyContent = null;
 						this.loadBody();

web/src/detailPanel.js

@@ -12,11 +12,23 @@ function init(React){
 			    id   = self.props.data.id;
 			if(!id) return;
 
-			ws.reqBody(id,function(content){
+			jQuery.get("/fetchBody?id=" + id ,function(resObj){
-				if(content.id == self.props.data.id){
+				if(resObj && resObj.id){
-					self.setState({
+					if(resObj.type && resObj.type == "image" && resObj.ref){
-						body : content
+						self.setState({
-					});
+							body : {
+								img : resObj.ref,
+								id  : resObj.id
+							}
+						});
+					}else if(resObj.content){
+						self.setState({
+							body : {
+								body : resObj.content,
+								id   : resObj.id
+							}
+						});
+					}
 				}
 			});
 		},
@@ -55,9 +67,13 @@ function init(React){
 			);
 
 			if(this.props.data.statusCode){
-
 				if(this.state.body.id == this.props.data.id){
-					bodyContent = (<pre className="resBodyContent">{this.state.body.body}</pre>);
+					if(this.state.body.img){
+						var imgEl = { __html : '<img src="'+ this.state.body.img +'" />'};
+						bodyContent = (<div dangerouslySetInnerHTML={imgEl}></div>);
+					}else{
+						bodyContent = (<pre className="resBodyContent">{this.state.body.body}</pre>);
+					}
 				}else{
 					bodyContent = null;
 					this.loadBody();