move response data from memory to local cache file

lib/recorder.js

@@ -2,15 +2,18 @@
 var zlib      = require('zlib'),
     Datastore = require('nedb'), 
     util      = require("util"),
+    path      = require("path"),
     fs        = require("fs"),
     events    = require('events'),
     iconv     = require('iconv-lite'),
+    proxyUtil = require("./util"),
     logUtil   = require("./log");
 
 //option.filename
 function Recorder(option){
     var self = this,
-        id   = 1,
+        id        = 1,
+        cachePath = proxyUtil.generateCacheDir(),
         db;
 
     option = option || {};
@@ -87,31 +90,40 @@ function Recorder(option){
     };
 
     //update recordBody if exits
+
+    //TODO : trigger update callback
+    var BODY_FILE_PRFIX = "res_body_";
     self.updateRecordBody =function(id,info){
         if(id == -1) return;
 
         if(!id || !info.resBody) return;
         //add to body map
         //ignore image data
-        if(/image/.test(info.resHeader['content-type'])){
-            self.recordBodyMap[id] = "(image)";
-        }else{
-            self.recordBodyMap[id] = info.resBody;
-        }
+        var bodyFile = path.join(cachePath,BODY_FILE_PRFIX + id);
+        fs.writeFile(bodyFile, info.resBody);
     };
 
-    self.getBody = function(id){
+    self.getBody = function(id,cb){
         if(id < 0){
-            return "";
+            cb && cb("");
         }
 
-        return self.recordBodyMap[id] || "";
+        var bodyFile = path.join(cachePath,BODY_FILE_PRFIX + id);
+        fs.access(bodyFile, fs.F_OK | fs.R_OK ,function(err){
+            if(err){
+                cb && cb(err);
+            }else{
+                fs.readFile(bodyFile,cb);
+            }
+        });
     };
 
-    self.getBodyUTF8 = function(id,cb){
-        var bodyContent = self.getBody(id),
-            result      = "";
-
+    self.getDecodedBody = function(id,cb){
+        var result = {
+            type    : "unknown",
+            mime    : "",
+            content : ""
+        };
         GLOBAL.recorder.getSingleRecord(id,function(err,doc){
             //check whether this record exists
             if(!doc || !doc[0]){
@@ -119,24 +131,39 @@ function Recorder(option){
                 return;
             }
 
-            if(!bodyContent){
-                cb(null,result);
-            }else{
-                var record      = doc[0],
-                    resHeader   = record['resHeader'] || {};
-                try{
-                    var charsetMatch = JSON.stringify(resHeader).match(/charset="?([a-zA-Z0-9\-]+)"?/);
-                    if(charsetMatch && charsetMatch.length > 1){
-                        var currentCharset = charsetMatch[1].toLowerCase();
-                        if(currentCharset != "utf-8" && iconv.encodingExists(currentCharset)){
-                            bodyContent = iconv.decode(bodyContent, currentCharset);
+            self.getBody(id,function(err,bodyContent){
+                if(err){
+                    cb(err);
+                }else if(!bodyContent){
+                    cb(null,result);
+                }else{
+                    var record      = doc[0],
+                        resHeader   = record['resHeader'] || {};
+                    try{
+                        var headerStr    = JSON.stringify(resHeader),
+                            charsetMatch = headerStr.match(/charset="?([a-zA-Z0-9\-]+)"?/),
+                            imageMatch   = resHeader && resHeader["content-type"];
+
+                        if(charsetMatch && charsetMatch.length){
+
+                            var currentCharset = charsetMatch[1].toLowerCase();
+                            if(currentCharset != "utf-8" && iconv.encodingExists(currentCharset)){
+                                bodyContent = iconv.decode(bodyContent, currentCharset);
+                            }
+                            result.type    = "text";
+                            result.content = bodyContent.toString();
+                        }else if(imageMatch){
+
+                            result.type    = "image";
+                            result.mime    = imageMatch;
+                            result.content = bodyContent;
                         }
-                    }
-                }catch(e){}
+                    }catch(e){}
 
-                cb(null,bodyContent.toString());
-            }
+                    cb(null,result);
+                }
 
+            });
         });
     };
 

lib/requestHandler.js

@@ -24,6 +24,7 @@ function userRequestHandler(req,userRes){
         in http  server : http://www.example.com/a/b/c
         in https server : /a/b/c
     */
+
     var host               = req.headers.host,
         protocol           = (!!req.connection.encrypted && !/^http:/.test(req.url)) ? "https" : "http",
         fullUrl            = protocol === "http" ? req.url : (protocol + '://' + host + req.url),
@@ -33,6 +34,9 @@ function userRequestHandler(req,userRes){
         resourceInfoId     = -1,
         reqData;
 
+    // console.log(req.url);
+    // console.log(path);
+
     //record
     resourceInfo = {
         host      : host,

lib/util.js

@@ -1,5 +1,7 @@
 var fs   = require("fs"),
-    path = require("path");
+    path = require("path"),
+    exec = require('child_process').exec;
+
 
 // {"Content-Encoding":"gzip"} --> {"content-encoding":"gzip"}
 module.exports.lower_keys = function(obj){
@@ -41,6 +43,27 @@ module.exports.getAnyProxyHome = function(){
     return home;
 }
 
+var CACHE_DIR_PREFIX = "cache_r";
+module.exports.generateCacheDir = function(){
+    var rand      = Math.floor(Math.random() * 1000000),
+        cachePath = path.join(util.getAnyProxyHome(),"./" + CACHE_DIR_PREFIX + rand);
+
+    fs.mkdirSync(cachePath,0777);
+    return cachePath;
+}
+
+module.exports.clearCacheDir = function(cb){
+    var home  = util.getAnyProxyHome(),
+        isWin = /^win/.test(process.platform);
+
+    var dirNameWildCard = CACHE_DIR_PREFIX + "*";
+    if(isWin){
+        exec("for /D %f in (" + dirNameWildCard + ") do rmdir %f /s /q",{cwd : home},cb);
+    }else{
+        exec("rm -rf " + dirNameWildCard + "",{cwd : home},cb);        
+    }
+}
+
 module.exports.simpleRender = function(str, object, regexp){
     return String(str).replace(regexp || (/\{\{([^{}]+)\}\}/g), function(match, name){
         if (match.charAt(0) == '\\') return match.slice(1);

lib/webInterface.js

@@ -48,6 +48,36 @@ function webInterface(config){
         });
     });
 
+    app.get("/fetchBody",function(req,res){
+        var query = req.query;
+        if(query && query.id){
+            GLOBAL.recorder.getDecodedBody(query.id, function(err, result){
+                if(err || !result || !result.content){
+                    res.json({});
+                }else if(result.type && result.type == "image" && result.mime){
+                    if(query.raw){
+                        //TODO : cache query result
+                        res.type(result.mime).end(result.content);
+                    }else{
+                        res.json({
+                            id      : query.id,
+                            type    : result.type,
+                            ref     : "/fetchBody?id=" + query.id + "&raw=true"
+                        });
+                    }
+                }else{
+                    res.json({
+                        id      : query.id,
+                        type    : result.type,
+                        content : result.content
+                    });
+                }
+            });
+        }else{
+            res.end({});
+        }
+    });
+
     app.get("/fetchCrtFile",function(req,res){
         if(crtFilePath){
             res.setHeader("Content-Type","application/x-x509-ca-cert");

lib/wsServer.js

@@ -23,8 +23,8 @@ function resToMsg(msg,cb){
     }
 
     if(jsonData.type == "reqBody" && jsonData.id){
-        result.type ="body";
-        GLOBAL.recorder.getBodyUTF8(jsonData.id, function(err, data){
+        result.type = "body";
+        GLOBAL.recorder.getBody(jsonData.id, function(err, data){
             if(err){
                 result.content = {
                     id    : null,
@@ -34,7 +34,7 @@ function resToMsg(msg,cb){
             }else{
                 result.content = {
                     id   : jsonData.id,
-                    body : data
+                    body : data.toString()
                 };
             }
             cb && cb(result);