perf(缓存): 优化缓存清理和服务逻辑,使用异步文件操作和并行处理
重构缓存清理逻辑,使用异步文件操作和并行删除提高效率 优化serveFromCache函数,增加缓存失效时的自动重试机制 使用流管道优化文件传输性能,增加缓冲区大小 移除未使用的sharp依赖和缩略图生成逻辑
This commit is contained in:
405
source.js
405
source.js
@@ -5,11 +5,10 @@ const querystring = require('querystring');
|
||||
const fs = require('fs');
|
||||
const pathModule = require('path');
|
||||
const crypto = require('crypto');
|
||||
const sharp = require('sharp');
|
||||
|
||||
const CACHE_DIR_NAME = '.cache';
|
||||
const DEFAULT_PORT = 9001;
|
||||
const DEFAULT_API_ENDPOINT = 'http://183.6.121.121:9521/alist';
|
||||
const DEFAULT_API_ENDPOINT = 'http://183.6.121.121:9519/api';
|
||||
|
||||
const cacheDir = pathModule.join(__dirname, CACHE_DIR_NAME);
|
||||
const pathIndex = {};
|
||||
@@ -80,13 +79,40 @@ const HTTP_STATUS = {
|
||||
};
|
||||
|
||||
// 定时清理过期缓存数据
|
||||
setInterval(() => {
|
||||
setInterval(async () => {
|
||||
const currentTime = Date.now();
|
||||
const keysToDelete = [];
|
||||
const filesToDelete = [];
|
||||
|
||||
// 第一步:收集需要删除的键和文件
|
||||
for (const key in pathIndex) {
|
||||
if (currentTime - pathIndex[key].timestamp > CACHE_EXPIRY_MS) {
|
||||
delete pathIndex[key];
|
||||
keysToDelete.push(key);
|
||||
const cacheMetaFile = pathModule.join(cacheDir, `${key}.meta`);
|
||||
const cacheContentFile = pathModule.join(cacheDir, `${pathIndex[key].uniqid}.content`);
|
||||
filesToDelete.push(cacheMetaFile, cacheContentFile);
|
||||
}
|
||||
}
|
||||
|
||||
// 第二步:从内存中删除过期索引
|
||||
keysToDelete.forEach(key => delete pathIndex[key]);
|
||||
|
||||
// 第三步:异步删除文件系统中的缓存文件
|
||||
if (filesToDelete.length > 0) {
|
||||
console.log(`Cleaning up ${keysToDelete.length} expired cache entries`);
|
||||
|
||||
// 并行删除文件,但限制并发数为10
|
||||
const deletePromises = filesToDelete.map(file =>
|
||||
fs.promises.unlink(file).catch(err => {
|
||||
if (err.code !== 'ENOENT') { // 忽略文件不存在的错误
|
||||
console.warn(`Failed to delete cache file ${file}:`, err.message);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
// 使用Promise.all处理所有删除操作
|
||||
await Promise.all(deletePromises);
|
||||
}
|
||||
}, CACHE_CLEANUP_INTERVAL_MS);
|
||||
|
||||
// 统一发送错误响应
|
||||
@@ -162,7 +188,7 @@ async function processSuccessfulApiData(apiData, uniqidhex, reqPath, token, sign
|
||||
console.warn(`Content length mismatch for ${cacheContentFile}. API: ${data.headers['content-length']}, Cache: ${contentLength}. Re-fetching.`);
|
||||
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||
} else {
|
||||
serveFromCache(data, cacheContentFile, cacheMetaFile, res);
|
||||
serveFromCache(data, cacheContentFile, cacheMetaFile, res, reqPath, token, sign, uniqidhex);
|
||||
}
|
||||
} else {
|
||||
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||
@@ -177,7 +203,7 @@ async function tryServeFromStaleCacheOrError(uniqidhex, res, errorMessage) {
|
||||
console.warn(`API call failed or returned non-200. Serving stale cache for ${uniqidhex}`);
|
||||
try {
|
||||
const cacheData = JSON.parse(fs.readFileSync(cacheMetaFile, 'utf8'));
|
||||
serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res);
|
||||
serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res, null, null, null, uniqidhex);
|
||||
return;
|
||||
} catch (parseError) {
|
||||
console.error(`Error parsing stale meta file ${cacheMetaFile}:`, parseError);
|
||||
@@ -225,7 +251,7 @@ async function handleMainRequest(req, res) {
|
||||
res.end();
|
||||
} else {
|
||||
viewsInfo.increment('cacheHit');
|
||||
serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res);
|
||||
serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res, reqPath, token, sign, uniqidhex);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
@@ -255,7 +281,7 @@ const server = http.createServer(handleMainRequest);
|
||||
// 检查缓存头并返回是否为304
|
||||
async function checkCacheHeaders(req, cacheMetaFile) {
|
||||
try {
|
||||
const metaContent = fs.readFileSync(cacheMetaFile, 'utf8');
|
||||
const metaContent = await fs.promises.readFile(cacheMetaFile, 'utf8');
|
||||
const cacheData = JSON.parse(metaContent);
|
||||
const ifNoneMatch = req.headers['if-none-match'];
|
||||
const ifModifiedSince = req.headers['if-modified-since'];
|
||||
@@ -283,22 +309,24 @@ async function checkCacheHeaders(req, cacheMetaFile) {
|
||||
return { cacheData, isNotModified: false };
|
||||
} catch (error) {
|
||||
console.error(`Error reading or parsing cache meta file ${cacheMetaFile} in checkCacheHeaders:`, error);
|
||||
// If we can't read meta, assume cache is invalid or treat as not modified: false
|
||||
// Returning a dummy cacheData or null might be better depending on how caller handles it.
|
||||
// For now, let it propagate and potentially fail later if cacheData is expected.
|
||||
// Or, more safely, indicate cache is not valid / not modified is false.
|
||||
return { cacheData: null, isNotModified: false }; // Indicate failure to load cacheData
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 检查缓存是否有效
|
||||
function isCacheValid(cacheMetaFile, cacheContentFile) {
|
||||
if (!fs.existsSync(cacheMetaFile) || !fs.existsSync(cacheContentFile)) {
|
||||
return false;
|
||||
}
|
||||
async function isCacheValid(cacheMetaFile, cacheContentFile) {
|
||||
try {
|
||||
const metaContent = fs.readFileSync(cacheMetaFile, 'utf8');
|
||||
// 使用Promise.all并行检查文件是否存在
|
||||
const [metaExists, contentExists] = await Promise.all([
|
||||
fs.promises.access(cacheMetaFile).then(() => true).catch(() => false),
|
||||
fs.promises.access(cacheContentFile).then(() => true).catch(() => false)
|
||||
]);
|
||||
|
||||
if (!metaExists || !contentExists) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const metaContent = await fs.promises.readFile(cacheMetaFile, 'utf8');
|
||||
const cacheData = JSON.parse(metaContent);
|
||||
// Ensure expiration is a number and in the future
|
||||
return typeof cacheData.expiration === 'number' && cacheData.expiration > Date.now();
|
||||
@@ -375,51 +403,34 @@ async function fetchApiData(reqPath, token, sign) {
|
||||
});
|
||||
}
|
||||
|
||||
// createThumbnail
|
||||
function createThumbnail(data, cacheContentFile) {
|
||||
const { path, thumb } = data;
|
||||
|
||||
const thumbCacheFile = pathModule.join(cacheDir, `thumb_${thumb.uniqid}.jpeg`);
|
||||
if (fs.existsSync(thumbCacheFile)) return thumbCacheFile;
|
||||
|
||||
const isVideo = path && typeof path === 'string' && path.includes('.mp4');
|
||||
if (isVideo || !thumb) return;
|
||||
const width = thumb.width && thumb.width > 0 ? thumb.width : undefined;
|
||||
const height = thumb.height && thumb.height > 0 ? thumb.height : undefined;
|
||||
if (!width) return;
|
||||
sharp(cacheContentFile).resize(width, height).toFile(thumbCacheFile);
|
||||
return thumbCacheFile;
|
||||
}
|
||||
|
||||
|
||||
// 从真实 URL 获取数据并写入缓存
|
||||
const REAL_URL_FETCH_TIMEOUT_MS = 0; // 0 means no timeout for the actual file download
|
||||
|
||||
const fetchAndServe = (data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res) => {
|
||||
const fetchAndServe = async (data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res) => {
|
||||
const protocol = data.realUrl.startsWith('https:') ? https : http;
|
||||
|
||||
protocol.get(data.realUrl, { timeout: REAL_URL_FETCH_TIMEOUT_MS, rejectUnauthorized: false }, (realRes) => {
|
||||
const cacheStream = fs.createWriteStream(tempCacheContentFile, { flags: 'w' });
|
||||
const cacheStream = fs.createWriteStream(tempCacheContentFile, { flags: 'w', highWaterMark: 64 * 1024 }); // 增加缓冲区大小到64KB
|
||||
|
||||
let isVideo = data.path && typeof data.path === 'string' && data.path.includes('.mp4');
|
||||
// 确保 content-length 是有效的
|
||||
const contentLength = realRes.headers['content-length'];
|
||||
if (contentLength) {
|
||||
|
||||
// contentLength 小于 2KB 且与缓存文件大小不一致时,重新获取
|
||||
if (contentLength < 2048 && data.headers['content-length'] !== contentLength) {
|
||||
console.warn('Warning: content-length is different for the response from:', data.realUrl);
|
||||
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, `Bad Gateway: Content-Length mismatch for ${data.realUrl}`);
|
||||
// Clean up temp file if stream hasn't started or failed early
|
||||
if (fs.existsSync(tempCacheContentFile)) {
|
||||
fs.unlinkSync(tempCacheContentFile);
|
||||
}
|
||||
fs.promises.access(tempCacheContentFile)
|
||||
.then(() => fs.promises.unlink(tempCacheContentFile))
|
||||
.catch(() => {}); // 忽略文件不存在的错误
|
||||
return;
|
||||
}
|
||||
|
||||
data.headers['content-length'] = contentLength;
|
||||
// 更新 data 到缓存 cacheMetaFile
|
||||
fs.writeFileSync(cacheMetaFile, JSON.stringify(data));
|
||||
// 异步更新 data 到缓存 cacheMetaFile
|
||||
fs.promises.writeFile(cacheMetaFile, JSON.stringify(data))
|
||||
.catch(err => console.error(`Error writing meta file ${cacheMetaFile}:`, err));
|
||||
} else {
|
||||
console.warn('Warning: content-length is undefined for the response from:', data.realUrl);
|
||||
}
|
||||
@@ -432,98 +443,192 @@ const fetchAndServe = (data, tempCacheContentFile, cacheContentFile, cacheMetaFi
|
||||
'Expires': new Date(Date.now() + 31536000000).toUTCString(),
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Connection': 'keep-alive',
|
||||
'Date': new Date().toUTCString(), // Should be set by the server, but good for consistency
|
||||
'Last-Modified': data.headers['last-modified'] || new Date(fs.statSync(cacheMetaFile).mtime).toUTCString(), // Prefer API's Last-Modified if available
|
||||
'Date': new Date().toUTCString(),
|
||||
'Last-Modified': data.headers['last-modified'] || new Date().toUTCString(),
|
||||
};
|
||||
const responseHeaders = {
|
||||
...baseHeaders,
|
||||
'Content-Type': realRes.headers['content-type'] || (isVideo ? 'video/mp4' : 'application/octet-stream'), // Prefer actual content-type
|
||||
...data.headers, // Allow API to override some headers if necessary
|
||||
'Content-Type': realRes.headers['content-type'] || (isVideo ? 'video/mp4' : 'application/octet-stream'),
|
||||
...data.headers,
|
||||
};
|
||||
|
||||
res.writeHead(realRes.statusCode, responseHeaders);
|
||||
realRes.pipe(cacheStream);
|
||||
realRes.pipe(res);
|
||||
|
||||
realRes.on('end', () => {
|
||||
cacheStream.end(() => { // Ensure stream is fully flushed before renaming
|
||||
if (fs.existsSync(tempCacheContentFile)) {
|
||||
try {
|
||||
// Ensure the target directory exists before renaming
|
||||
const targetDir = pathModule.dirname(cacheContentFile);
|
||||
if (!fs.existsSync(targetDir)) {
|
||||
fs.mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
fs.renameSync(tempCacheContentFile, cacheContentFile);
|
||||
console.log(`Successfully cached: ${cacheContentFile}`);
|
||||
|
||||
// 生成缩略图
|
||||
if (data.thumb) {
|
||||
createThumbnail(data, cacheContentFile);
|
||||
}
|
||||
} catch (renameError) {
|
||||
console.error(`Error renaming temp cache file ${tempCacheContentFile} to ${cacheContentFile}:`, renameError);
|
||||
// If rename fails, try to remove the temp file to avoid clutter
|
||||
try { fs.unlinkSync(tempCacheContentFile); } catch (e) { /* ignore */ }
|
||||
}
|
||||
} else {
|
||||
// This case might indicate an issue if the stream ended but no temp file was created/found
|
||||
console.warn(`Temp cache file ${tempCacheContentFile} not found after stream end for ${data.realUrl}`);
|
||||
|
||||
// 使用管道优化流传输
|
||||
const pipeline = require('stream').pipeline;
|
||||
|
||||
// 创建一个流分支,同时写入缓存和响应
|
||||
const { PassThrough } = require('stream');
|
||||
const passThrough = new PassThrough();
|
||||
|
||||
passThrough.pipe(cacheStream);
|
||||
passThrough.pipe(res);
|
||||
|
||||
// 使用pipeline处理流错误
|
||||
pipeline(
|
||||
realRes,
|
||||
passThrough,
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.error(`Pipeline error for ${data.realUrl}:`, err);
|
||||
handleResponseError(res, tempCacheContentFile, data.realUrl);
|
||||
return;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
realRes.on('error', (streamError) => {
|
||||
console.error(`Error during response stream from ${data.realUrl}:`, streamError);
|
||||
cacheStream.end(); // Close the writable stream
|
||||
handleResponseError(res, tempCacheContentFile, data.realUrl); // tempCacheContentFile might be partially written
|
||||
});
|
||||
|
||||
|
||||
// 流处理完成后,重命名临时文件
|
||||
fs.promises.access(tempCacheContentFile)
|
||||
.then(() => {
|
||||
// 确保目标目录存在
|
||||
return fs.promises.mkdir(pathModule.dirname(cacheContentFile), { recursive: true })
|
||||
.then(() => fs.promises.rename(tempCacheContentFile, cacheContentFile))
|
||||
.then(() => console.log(`Successfully cached: ${cacheContentFile}`))
|
||||
.catch(renameError => {
|
||||
console.error(`Error renaming temp cache file ${tempCacheContentFile} to ${cacheContentFile}:`, renameError);
|
||||
return fs.promises.unlink(tempCacheContentFile).catch(() => {});
|
||||
});
|
||||
})
|
||||
.catch(() => {
|
||||
console.warn(`Temp cache file ${tempCacheContentFile} not found after stream end for ${data.realUrl}`);
|
||||
});
|
||||
}
|
||||
);
|
||||
}).on('error', (requestError) => {
|
||||
console.error(`Error making GET request to ${data.realUrl}:`, requestError);
|
||||
// No cacheStream involved here if the request itself fails before response
|
||||
handleResponseError(res, tempCacheContentFile, data.realUrl); // tempCacheContentFile might not exist or be empty
|
||||
handleResponseError(res, tempCacheContentFile, data.realUrl);
|
||||
});
|
||||
};
|
||||
|
||||
// 从缓存中读取数据并返回
|
||||
function serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res) {
|
||||
if (!cacheData) { // Added check for null cacheData from checkCacheHeaders failure
|
||||
console.error(`serveFromCache called with null cacheData for ${cacheContentFile}`);
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Cache metadata unavailable.');
|
||||
return;
|
||||
async function serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res, reqPath, token, sign, uniqidhex) {
|
||||
if (!cacheData) { // 缓存数据不可用,尝试重新获取
|
||||
console.warn(`Cache metadata unavailable for ${cacheContentFile}, attempting to fetch fresh data`);
|
||||
|
||||
// 如果提供了请求参数,尝试重新获取数据
|
||||
if (reqPath && token) {
|
||||
try {
|
||||
viewsInfo.increment('apiCall');
|
||||
const apiData = await fetchApiData(reqPath, token, sign);
|
||||
|
||||
if (apiData.code === HTTP_STATUS.REDIRECT || apiData.code === 301) {
|
||||
res.writeHead(HTTP_STATUS.REDIRECT, { Location: apiData.data.url });
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (apiData.code === HTTP_STATUS.OK && apiData.data && apiData.data.url) {
|
||||
const { url: realUrl, cloudtype, expiration, path: apiPath, headers, uniqid, thumb } = apiData.data;
|
||||
const data = { realUrl, cloudtype, expiration: expiration * 1000, path: apiPath, headers, uniqid, thumb };
|
||||
|
||||
// 更新索引
|
||||
pathIndex[uniqidhex] = { uniqid: data.uniqid, timestamp: Date.now() };
|
||||
|
||||
// 写入新的元数据
|
||||
await fs.promises.mkdir(pathModule.dirname(cacheMetaFile), { recursive: true });
|
||||
await fs.promises.writeFile(cacheMetaFile, JSON.stringify(data));
|
||||
|
||||
// 获取并提供新数据
|
||||
const tempCacheContentFile = pathModule.join(cacheDir, `${data.uniqid}_${crypto.randomBytes(16).toString('hex')}.temp`);
|
||||
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||
return;
|
||||
} else {
|
||||
viewsInfo.increment('fetchApiWarning');
|
||||
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, apiData.message || 'Failed to fetch data from API');
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
viewsInfo.increment('fetchApiError');
|
||||
console.error('Error fetching fresh data:', error);
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, `Failed to fetch fresh data: ${error.message}`);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// 如果没有提供请求参数,无法重新获取
|
||||
console.error(`serveFromCache called with null cacheData and insufficient request info for ${cacheContentFile}`);
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Cache metadata unavailable and cannot fetch fresh data.');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 使用异步方式获取ETag和Last-Modified
|
||||
let etag = cacheData.uniqid;
|
||||
let lastModified = cacheData.headers && cacheData.headers['last-modified'];
|
||||
|
||||
if (!etag || !lastModified) {
|
||||
try {
|
||||
const [fileStats, metaStats] = await Promise.all([
|
||||
fs.promises.stat(cacheContentFile).catch(() => null),
|
||||
fs.promises.stat(cacheMetaFile).catch(() => null)
|
||||
]);
|
||||
|
||||
if (!etag && fileStats) {
|
||||
// 使用文件大小和修改时间作为ETag的一部分,避免读取整个文件计算MD5
|
||||
etag = crypto.createHash('md5')
|
||||
.update(`${fileStats.size}-${fileStats.mtime.getTime()}`)
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
if (!lastModified && metaStats) {
|
||||
lastModified = new Date(metaStats.mtime).toUTCString();
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Error getting file stats for cache: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
const baseHeaders = {
|
||||
'Cloud-Type': cacheData.cloudtype || 'unknown',
|
||||
'Cloud-Expiration': cacheData.expiration || 'N/A',
|
||||
'ETag': cacheData.uniqid || crypto.createHash('md5').update(fs.readFileSync(cacheContentFile)).digest('hex'), // Fallback ETag if missing
|
||||
'ETag': etag || '',
|
||||
'Cache-Control': 'public, max-age=31536000', // 1 year
|
||||
'Expires': new Date(Date.now() + 31536000000).toUTCString(),
|
||||
'Accept-Ranges': 'bytes',
|
||||
'Connection': 'keep-alive',
|
||||
'Date': new Date().toUTCString(),
|
||||
'Last-Modified': (cacheData.headers && cacheData.headers['last-modified']) || new Date(fs.statSync(cacheMetaFile).mtime).toUTCString(),
|
||||
'Last-Modified': lastModified || new Date().toUTCString(),
|
||||
};
|
||||
if (cacheData.thumb) {
|
||||
var thumbCacheFile = createThumbnail(cacheData, cacheContentFile)
|
||||
if (thumbCacheFile && fs.existsSync(thumbCacheFile)) {
|
||||
cacheData.headers['content-length'] = fs.statSync(thumbCacheFile).size;
|
||||
const responseHeaders = {
|
||||
...baseHeaders,
|
||||
...(cacheData.headers || {}),
|
||||
'ETag': (cacheData.thumb.uniqid || cacheData.uniqid) + '_thumb',
|
||||
'Content-Type': 'image/jpeg',
|
||||
};
|
||||
res.writeHead(HTTP_STATUS.OK, responseHeaders);
|
||||
const thumbStream = fs.createReadStream(thumbCacheFile);
|
||||
thumbStream.pipe(res);
|
||||
|
||||
viewsInfo.increment('cacheCall');
|
||||
|
||||
// 先检查缓存文件是否存在且可读
|
||||
try {
|
||||
await fs.promises.access(cacheContentFile, fs.constants.R_OK);
|
||||
} catch (error) {
|
||||
console.warn(`Cache content file ${cacheContentFile} not accessible: ${error.message}`);
|
||||
|
||||
// 如果提供了请求参数,尝试重新获取数据
|
||||
if (reqPath && token) {
|
||||
console.log(`Attempting to fetch fresh data for ${cacheContentFile}`);
|
||||
try {
|
||||
viewsInfo.increment('apiCall');
|
||||
const apiData = await fetchApiData(reqPath, token, sign);
|
||||
|
||||
if (apiData.code === HTTP_STATUS.OK && apiData.data && apiData.data.url) {
|
||||
const { url: realUrl, cloudtype, expiration, path: apiPath, headers, uniqid, thumb } = apiData.data;
|
||||
const data = { realUrl, cloudtype, expiration: expiration * 1000, path: apiPath, headers, uniqid, thumb };
|
||||
|
||||
// 更新索引
|
||||
pathIndex[uniqidhex] = { uniqid: data.uniqid, timestamp: Date.now() };
|
||||
|
||||
// 获取并提供新数据
|
||||
const tempCacheContentFile = pathModule.join(cacheDir, `${data.uniqid}_${crypto.randomBytes(16).toString('hex')}.temp`);
|
||||
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||
return;
|
||||
} else {
|
||||
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, apiData.message || 'Failed to fetch data from API');
|
||||
return;
|
||||
}
|
||||
} catch (fetchError) {
|
||||
console.error(`Error fetching fresh data: ${fetchError.message}`);
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, `Failed to fetch fresh data: ${fetchError.message}`);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Unable to read cache content file and cannot fetch fresh data.');
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
viewsInfo.increment('cacheCall');
|
||||
const readStream = fs.createReadStream(cacheContentFile);
|
||||
|
||||
const readStream = fs.createReadStream(cacheContentFile, { highWaterMark: 64 * 1024 }); // 增加读取缓冲区大小
|
||||
const isVideo = cacheData.path && typeof cacheData.path === 'string' && cacheData.path.includes('.mp4');
|
||||
|
||||
let currentContentLength = cacheData.headers && cacheData.headers['content-length'] ? parseInt(cacheData.headers['content-length'], 10) : 0;
|
||||
@@ -544,7 +649,7 @@ function serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res) {
|
||||
}
|
||||
} catch (statError) {
|
||||
console.error(`Error stating cache content file ${cacheContentFile}:`, statError);
|
||||
handleCacheReadError(res, cacheContentFile); // Treat stat error as read error
|
||||
handleCacheReadError(res, cacheContentFile, reqPath, token, sign, uniqidhex); // Treat stat error as read error
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -565,7 +670,38 @@ function serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res) {
|
||||
|
||||
readStream.on('error', (err) => {
|
||||
console.error(`Read stream error for ${cacheContentFile}:`, err);
|
||||
handleCacheReadError(res, cacheContentFile);
|
||||
|
||||
// 如果提供了请求参数,尝试重新获取数据而不是直接报错
|
||||
if (reqPath && token) {
|
||||
console.log(`Read stream error, attempting to fetch fresh data for ${cacheContentFile}`);
|
||||
viewsInfo.increment('apiCall');
|
||||
|
||||
fetchApiData(reqPath, token, sign)
|
||||
.then(apiData => {
|
||||
if (apiData.code === HTTP_STATUS.OK && apiData.data && apiData.data.url) {
|
||||
const { url: realUrl, cloudtype, expiration, path: apiPath, headers, uniqid, thumb } = apiData.data;
|
||||
const data = { realUrl, cloudtype, expiration: expiration * 1000, path: apiPath, headers, uniqid, thumb };
|
||||
|
||||
// 更新索引
|
||||
pathIndex[uniqidhex] = { uniqid: data.uniqid, timestamp: Date.now() };
|
||||
|
||||
// 获取并提供新数据
|
||||
const tempCacheContentFile = pathModule.join(cacheDir, `${data.uniqid}_${crypto.randomBytes(16).toString('hex')}.temp`);
|
||||
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||
} else {
|
||||
viewsInfo.increment('fetchApiWarning');
|
||||
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, apiData.message || 'Failed to fetch data from API');
|
||||
}
|
||||
})
|
||||
.catch(fetchError => {
|
||||
viewsInfo.increment('fetchApiError');
|
||||
console.error(`Error fetching fresh data after read stream error: ${fetchError.message}`);
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, `Failed to fetch fresh data: ${fetchError.message}`);
|
||||
});
|
||||
} else {
|
||||
// 如果没有提供请求参数,使用原始的错误处理
|
||||
handleCacheReadError(res, cacheContentFile, reqPath, token, sign, uniqidhex);
|
||||
}
|
||||
});
|
||||
|
||||
// Handle cases where client closes connection prematurely
|
||||
@@ -593,10 +729,55 @@ const handleResponseError = (res, tempCacheContentFile, realUrl) => {
|
||||
};
|
||||
|
||||
// 处理缓存读取错误
|
||||
const handleCacheReadError = (res, filePath) => {
|
||||
const handleCacheReadError = (res, filePath, reqPath, token, sign, uniqidhex) => {
|
||||
viewsInfo.increment('cacheReadError');
|
||||
console.error(`Error reading cache file: ${filePath}`);
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Internal Server Error: Unable to read cache content file');
|
||||
|
||||
// 如果提供了请求参数,尝试重新获取数据
|
||||
if (reqPath && token) {
|
||||
console.log(`Cache read error, attempting to fetch fresh data for ${filePath}`);
|
||||
viewsInfo.increment('apiCall');
|
||||
|
||||
fetchApiData(reqPath, token, sign)
|
||||
.then(apiData => {
|
||||
if (apiData.code === HTTP_STATUS.OK && apiData.data && apiData.data.url) {
|
||||
const { url: realUrl, cloudtype, expiration, path: apiPath, headers, uniqid, thumb } = apiData.data;
|
||||
const data = { realUrl, cloudtype, expiration: expiration * 1000, path: apiPath, headers, uniqid, thumb };
|
||||
|
||||
// 更新索引
|
||||
if (uniqidhex) {
|
||||
pathIndex[uniqidhex] = { uniqid: data.uniqid, timestamp: Date.now() };
|
||||
}
|
||||
|
||||
// 获取并提供新数据
|
||||
const cacheMetaFile = pathModule.join(cacheDir, `${uniqidhex}.meta`);
|
||||
const cacheContentFile = pathModule.join(cacheDir, `${data.uniqid}.content`);
|
||||
const tempCacheContentFile = pathModule.join(cacheDir, `${data.uniqid}_${crypto.randomBytes(16).toString('hex')}.temp`);
|
||||
|
||||
// 写入新的元数据
|
||||
fs.promises.mkdir(pathModule.dirname(cacheMetaFile), { recursive: true })
|
||||
.then(() => fs.promises.writeFile(cacheMetaFile, JSON.stringify(data)))
|
||||
.then(() => {
|
||||
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||
})
|
||||
.catch(writeError => {
|
||||
console.error(`Error writing meta file after cache read error: ${writeError.message}`);
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Failed to write cache metadata');
|
||||
});
|
||||
} else {
|
||||
viewsInfo.increment('fetchApiWarning');
|
||||
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, apiData.message || 'Failed to fetch data from API');
|
||||
}
|
||||
})
|
||||
.catch(fetchError => {
|
||||
viewsInfo.increment('fetchApiError');
|
||||
console.error(`Error fetching fresh data after cache read error: ${fetchError.message}`);
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, `Failed to fetch fresh data: ${fetchError.message}`);
|
||||
});
|
||||
} else {
|
||||
// 如果没有提供请求参数,返回错误
|
||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Internal Server Error: Unable to read cache content file');
|
||||
}
|
||||
};
|
||||
|
||||
// 启动服务器
|
||||
|
||||
Reference in New Issue
Block a user