mirror of
https://github.com/alibaba/anyproxy.git
synced 2025-04-22 05:00:59 +00:00
399 lines
11 KiB
TypeScript
399 lines
11 KiB
TypeScript
'use strict';
|
|
|
|
import * as Datastore from 'nedb';
|
|
import * as path from 'path';
|
|
import * as fs from 'fs';
|
|
import * as events from 'events';
|
|
import * as iconv from 'iconv-lite';
|
|
import * as fastJson from 'fast-json-stringify';
|
|
import logUtil from './log';
|
|
import proxyUtil from './util';
|
|
|
|
// //start recording and share a list when required
|
|
// const Datastore = require('nedb'),
|
|
// path = require('path'),
|
|
// fs = require('fs'),
|
|
// logUtil = require('./log'),
|
|
// events = require('events'),
|
|
// iconv = require('iconv-lite'),
|
|
// fastJson = require('fast-json-stringify'),
|
|
// proxyUtil = require('./util').default;
|
|
|
|
declare interface ISingleRecord {
|
|
_id?: number;
|
|
id?: number;
|
|
url?: string;
|
|
host?: string;
|
|
path?: string;
|
|
method?: string;
|
|
reqHeader?: OneLevelObjectType;
|
|
startTime?: number;
|
|
reqBody?: string;
|
|
protocol?: string;
|
|
statusCode?: number | string;
|
|
endTime?: number | string;
|
|
resHeader?: OneLevelObjectType;
|
|
length?: number | string;
|
|
mime?: string;
|
|
duration?: number | string;
|
|
}
|
|
|
|
|
|
const wsMessageStingify = fastJson({
|
|
title: 'ws message stringify',
|
|
type: 'object',
|
|
properties: {
|
|
time: {
|
|
type: 'integer',
|
|
},
|
|
message: {
|
|
type: 'string',
|
|
},
|
|
isToServer: {
|
|
type: 'boolean',
|
|
},
|
|
},
|
|
});
|
|
|
|
const BODY_FILE_PRFIX = 'res_body_';
|
|
const WS_MESSAGE_FILE_PRFIX = 'ws_message_';
|
|
const CACHE_DIR_PREFIX = 'cache_r';
|
|
function getCacheDir(): string {
|
|
const rand = Math.floor(Math.random() * 1000000);
|
|
const cachePath = path.join(proxyUtil.getAnyProxyPath('cache'), './' + CACHE_DIR_PREFIX + rand);
|
|
|
|
fs.mkdirSync(cachePath);
|
|
return cachePath;
|
|
}
|
|
|
|
function normalizeInfo(id: number, info: AnyProxyRecorder.ResourceInfo): ISingleRecord {
|
|
const singleRecord: ISingleRecord = {};
|
|
|
|
// general
|
|
singleRecord._id = id;
|
|
singleRecord.id = id;
|
|
singleRecord.url = info.url;
|
|
singleRecord.host = info.host;
|
|
singleRecord.path = info.path;
|
|
singleRecord.method = info.method;
|
|
|
|
// req
|
|
singleRecord.reqHeader = info.req.headers;
|
|
singleRecord.startTime = info.startTime;
|
|
singleRecord.reqBody = info.reqBody || '';
|
|
singleRecord.protocol = info.protocol || '';
|
|
|
|
// res
|
|
if (info.endTime) {
|
|
singleRecord.statusCode = info.statusCode;
|
|
singleRecord.endTime = info.endTime;
|
|
singleRecord.resHeader = info.resHeader;
|
|
singleRecord.length = info.length;
|
|
const contentType = info.resHeader['content-type'] || info.resHeader['Content-Type'];
|
|
if (contentType) {
|
|
singleRecord.mime = contentType.split(';')[0];
|
|
} else {
|
|
singleRecord.mime = '';
|
|
}
|
|
|
|
singleRecord.duration = info.endTime - info.startTime;
|
|
} else {
|
|
singleRecord.statusCode = '';
|
|
singleRecord.endTime = '';
|
|
singleRecord.resHeader = {};
|
|
singleRecord.length = '';
|
|
singleRecord.mime = '';
|
|
singleRecord.duration = '';
|
|
}
|
|
|
|
return singleRecord;
|
|
}
|
|
|
|
class Recorder extends events.EventEmitter {
|
|
private globalId: number;
|
|
private cachePath: string;
|
|
private db: Datastore;
|
|
constructor() {
|
|
super();
|
|
this.globalId = 1;
|
|
this.cachePath = getCacheDir();
|
|
this.db = new Datastore();
|
|
this.db.persistence.setAutocompactionInterval(5001);
|
|
|
|
// this.recordBodyMap = []; // id - body
|
|
}
|
|
|
|
public emitUpdate(id: number, info?: ISingleRecord): void {
|
|
const self = this;
|
|
if (info) {
|
|
self.emit('update', info);
|
|
} else {
|
|
self.getSingleRecord(id, (err, doc) => {
|
|
if (!err && !!doc && !!doc[0]) {
|
|
self.emit('update', doc[0]);
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
public emitUpdateLatestWsMessage(id: number, message: {
|
|
id: number,
|
|
message: AnyProxyRecorder.WsResourceInfo,
|
|
}): void {
|
|
this.emit('updateLatestWsMsg', message);
|
|
}
|
|
|
|
public updateRecord(id: number, info: AnyProxyRecorder.ResourceInfo): void {
|
|
if (id < 0) {
|
|
return;
|
|
}
|
|
const self = this;
|
|
const db = self.db;
|
|
|
|
const finalInfo = normalizeInfo(id, info);
|
|
|
|
db.update({ _id: id }, finalInfo);
|
|
self.updateRecordBody(id, info);
|
|
|
|
self.emitUpdate(id, finalInfo);
|
|
}
|
|
|
|
/**
|
|
* This method shall be called at each time there are new message
|
|
*
|
|
*/
|
|
public updateRecordWsMessage(id: number, message: AnyProxyRecorder.WsResourceInfo): void {
|
|
const cachePath = this.cachePath;
|
|
if (id < 0) {
|
|
return;
|
|
}
|
|
try {
|
|
const recordWsMessageFile = path.join(cachePath, WS_MESSAGE_FILE_PRFIX + id);
|
|
|
|
fs.appendFile(recordWsMessageFile, wsMessageStingify(message) + ',', (err) => {
|
|
if (err) {
|
|
logUtil.error(err.message);
|
|
}
|
|
});
|
|
} catch (e) {
|
|
console.error(e);
|
|
logUtil.error(e.message + e.stack);
|
|
}
|
|
|
|
this.emitUpdateLatestWsMessage(id, {
|
|
id,
|
|
message,
|
|
});
|
|
}
|
|
|
|
// public updateExtInfo(id: number , extInfo: any): void {
|
|
// const self = this;
|
|
// const db = self.db;
|
|
|
|
// db.update({ _id: id }, { $set: { ext: extInfo } }, {}, (err, nums) => {
|
|
// if (!err) {
|
|
// self.emitUpdate(id);
|
|
// }
|
|
// });
|
|
// }
|
|
|
|
public appendRecord(info: AnyProxyRecorder.ResourceInfo): number {
|
|
if (info.req.headers.anyproxy_web_req) { // TODO request from web interface
|
|
return -1;
|
|
}
|
|
const self = this;
|
|
const db = self.db;
|
|
|
|
const thisId = self.globalId++;
|
|
const finalInfo = normalizeInfo(thisId, info);
|
|
db.insert(finalInfo);
|
|
self.updateRecordBody(thisId, info);
|
|
|
|
self.emitUpdate(thisId, finalInfo);
|
|
return thisId;
|
|
}
|
|
|
|
public updateRecordBody(id: number, info: AnyProxyRecorder.ResourceInfo): void {
|
|
const self = this;
|
|
const cachePath = self.cachePath;
|
|
|
|
if (id === -1) {
|
|
return;
|
|
}
|
|
|
|
if (!id || typeof info.resBody === 'undefined') {
|
|
return;
|
|
}
|
|
// add to body map
|
|
// ignore image data
|
|
const bodyFile = path.join(cachePath, BODY_FILE_PRFIX + id);
|
|
fs.writeFile(bodyFile, info.resBody, (err) => {
|
|
if (err) {
|
|
logUtil.error(err.name);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* get body and websocket file
|
|
*
|
|
*/
|
|
public getBody(id: number, cb: (err: Error, content?: Buffer | string) => void): void {
|
|
const self = this;
|
|
const cachePath = self.cachePath;
|
|
|
|
if (id < 0) {
|
|
cb && cb(null, '');
|
|
}
|
|
|
|
const bodyFile = path.join(cachePath, BODY_FILE_PRFIX + id);
|
|
// node exported the `constants` from fs to maintain all the state constans since V7
|
|
// but the property `constants` does not exists in versions below 7, so we keep the way
|
|
fs.access(bodyFile, (fs as any).F_OK || (fs as any).R_OK, (err) => {
|
|
if (err) {
|
|
cb && cb(err);
|
|
} else {
|
|
fs.readFile(bodyFile, cb);
|
|
}
|
|
});
|
|
}
|
|
|
|
public getDecodedBody(id: number, cb: (err: Error, result?: {
|
|
method?: string;
|
|
type?: string;
|
|
mime?: string;
|
|
content?: string;
|
|
fileName?: string;
|
|
statusCode?: number;
|
|
}) => void): void {
|
|
const self = this;
|
|
const result = {
|
|
method: '',
|
|
type: 'unknown',
|
|
mime: '',
|
|
content: '',
|
|
fileName: undefined,
|
|
statusCode: undefined,
|
|
};
|
|
self.getSingleRecord(id, (err, doc) => {
|
|
// check whether this record exists
|
|
if (!doc || !doc[0]) {
|
|
cb(new Error('failed to find record for this id'));
|
|
return;
|
|
}
|
|
|
|
// also put the `method` back, so the client can decide whether to load ws messages
|
|
result.method = doc[0].method;
|
|
|
|
self.getBody(id, (error, bodyContent) => {
|
|
if (error) {
|
|
cb(error);
|
|
} else if (!bodyContent) {
|
|
cb(null, result);
|
|
} else {
|
|
const record = doc[0];
|
|
const resHeader = record.resHeader || {};
|
|
try {
|
|
const headerStr = JSON.stringify(resHeader);
|
|
const charsetMatch = headerStr.match(/charset='?([a-zA-Z0-9-]+)'?/);
|
|
const contentType = resHeader && (resHeader['content-type'] || resHeader['Content-Type']);
|
|
|
|
if (charsetMatch && charsetMatch.length) {
|
|
const currentCharset = charsetMatch[1].toLowerCase();
|
|
if (currentCharset !== 'utf-8' && iconv.encodingExists(currentCharset)) {
|
|
result.content = iconv.decode((bodyContent as Buffer), currentCharset);
|
|
} else {
|
|
result.content = bodyContent.toString();
|
|
}
|
|
|
|
result.mime = contentType;
|
|
result.type = contentType && /application\/json/i.test(contentType) ? 'json' : 'text';
|
|
} else if (contentType && /image/i.test(contentType)) {
|
|
result.type = 'image';
|
|
result.mime = contentType;
|
|
result.content = (bodyContent as string);
|
|
} else {
|
|
result.type = contentType;
|
|
result.mime = contentType;
|
|
result.content = bodyContent.toString();
|
|
}
|
|
result.fileName = path.basename(record.path);
|
|
result.statusCode = record.statusCode;
|
|
} catch (e) {
|
|
console.error(e);
|
|
}
|
|
cb(null, result);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* get decoded WebSoket messages
|
|
*
|
|
*/
|
|
public getDecodedWsMessage(id: number, cb: (err: Error, messages?: AnyProxyRecorder.WsResourceInfo[]) => void): void {
|
|
const self = this;
|
|
const cachePath = self.cachePath;
|
|
|
|
if (id < 0) {
|
|
cb && cb(null, []);
|
|
}
|
|
|
|
const wsMessageFile = path.join(cachePath, WS_MESSAGE_FILE_PRFIX + id);
|
|
fs.access(wsMessageFile, (fs as any).F_OK || (fs as any).R_OK, (err) => {
|
|
if (err) {
|
|
cb && cb(err);
|
|
} else {
|
|
fs.readFile(wsMessageFile, 'utf8', (error, content) => {
|
|
if (error) {
|
|
cb && cb(err);
|
|
}
|
|
|
|
try {
|
|
// remove the last dash "," if it has, since it's redundant
|
|
// and also add brackets to make it a complete JSON structure
|
|
content = `[${content.replace(/,$/, '')}]`;
|
|
const messages = JSON.parse(content);
|
|
cb(null, messages);
|
|
} catch (e) {
|
|
console.error(e);
|
|
logUtil.error(e.message + e.stack);
|
|
cb(e);
|
|
}
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
public getSingleRecord(id: number, cb: (err: Error, result: ISingleRecord) => void): void {
|
|
const self = this;
|
|
const db = self.db;
|
|
db.find({ _id: id }, cb);
|
|
}
|
|
|
|
public getSummaryList(cb: (err: Error, records: ISingleRecord[]) => void): void {
|
|
const self = this;
|
|
const db = self.db;
|
|
db.find({}, cb);
|
|
}
|
|
|
|
public getRecords(idStart: number | string, limit: number, cb: (err: Error, records: ISingleRecord[]) => void): void {
|
|
const self = this;
|
|
const db = self.db;
|
|
limit = limit || 10;
|
|
idStart = typeof idStart === 'number' ? idStart : (self.globalId - limit);
|
|
db.find({ _id: { $gte: idStart } })
|
|
.sort({ _id: 1 })
|
|
.limit(limit)
|
|
.exec(cb);
|
|
}
|
|
|
|
public clear(): void {
|
|
const self = this;
|
|
proxyUtil.deleteFolderContentsRecursive(self.cachePath, true);
|
|
}
|
|
}
|
|
|
|
export default Recorder;
|