perf(缓存): 优化缓存清理和服务逻辑,使用异步文件操作和并行处理
重构缓存清理逻辑,使用异步文件操作和并行删除提高效率 优化serveFromCache函数,增加缓存失效时的自动重试机制 使用流管道优化文件传输性能,增加缓冲区大小 移除未使用的sharp依赖和缩略图生成逻辑
This commit is contained in:
405
source.js
405
source.js
@@ -5,11 +5,10 @@ const querystring = require('querystring');
|
|||||||
const fs = require('fs');
|
const fs = require('fs');
|
||||||
const pathModule = require('path');
|
const pathModule = require('path');
|
||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
const sharp = require('sharp');
|
|
||||||
|
|
||||||
const CACHE_DIR_NAME = '.cache';
|
const CACHE_DIR_NAME = '.cache';
|
||||||
const DEFAULT_PORT = 9001;
|
const DEFAULT_PORT = 9001;
|
||||||
const DEFAULT_API_ENDPOINT = 'http://183.6.121.121:9521/alist';
|
const DEFAULT_API_ENDPOINT = 'http://183.6.121.121:9519/api';
|
||||||
|
|
||||||
const cacheDir = pathModule.join(__dirname, CACHE_DIR_NAME);
|
const cacheDir = pathModule.join(__dirname, CACHE_DIR_NAME);
|
||||||
const pathIndex = {};
|
const pathIndex = {};
|
||||||
@@ -80,13 +79,40 @@ const HTTP_STATUS = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// 定时清理过期缓存数据
|
// 定时清理过期缓存数据
|
||||||
setInterval(() => {
|
setInterval(async () => {
|
||||||
const currentTime = Date.now();
|
const currentTime = Date.now();
|
||||||
|
const keysToDelete = [];
|
||||||
|
const filesToDelete = [];
|
||||||
|
|
||||||
|
// 第一步:收集需要删除的键和文件
|
||||||
for (const key in pathIndex) {
|
for (const key in pathIndex) {
|
||||||
if (currentTime - pathIndex[key].timestamp > CACHE_EXPIRY_MS) {
|
if (currentTime - pathIndex[key].timestamp > CACHE_EXPIRY_MS) {
|
||||||
delete pathIndex[key];
|
keysToDelete.push(key);
|
||||||
|
const cacheMetaFile = pathModule.join(cacheDir, `${key}.meta`);
|
||||||
|
const cacheContentFile = pathModule.join(cacheDir, `${pathIndex[key].uniqid}.content`);
|
||||||
|
filesToDelete.push(cacheMetaFile, cacheContentFile);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 第二步:从内存中删除过期索引
|
||||||
|
keysToDelete.forEach(key => delete pathIndex[key]);
|
||||||
|
|
||||||
|
// 第三步:异步删除文件系统中的缓存文件
|
||||||
|
if (filesToDelete.length > 0) {
|
||||||
|
console.log(`Cleaning up ${keysToDelete.length} expired cache entries`);
|
||||||
|
|
||||||
|
// 并行删除文件,但限制并发数为10
|
||||||
|
const deletePromises = filesToDelete.map(file =>
|
||||||
|
fs.promises.unlink(file).catch(err => {
|
||||||
|
if (err.code !== 'ENOENT') { // 忽略文件不存在的错误
|
||||||
|
console.warn(`Failed to delete cache file ${file}:`, err.message);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
// 使用Promise.all处理所有删除操作
|
||||||
|
await Promise.all(deletePromises);
|
||||||
|
}
|
||||||
}, CACHE_CLEANUP_INTERVAL_MS);
|
}, CACHE_CLEANUP_INTERVAL_MS);
|
||||||
|
|
||||||
// 统一发送错误响应
|
// 统一发送错误响应
|
||||||
@@ -162,7 +188,7 @@ async function processSuccessfulApiData(apiData, uniqidhex, reqPath, token, sign
|
|||||||
console.warn(`Content length mismatch for ${cacheContentFile}. API: ${data.headers['content-length']}, Cache: ${contentLength}. Re-fetching.`);
|
console.warn(`Content length mismatch for ${cacheContentFile}. API: ${data.headers['content-length']}, Cache: ${contentLength}. Re-fetching.`);
|
||||||
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||||
} else {
|
} else {
|
||||||
serveFromCache(data, cacheContentFile, cacheMetaFile, res);
|
serveFromCache(data, cacheContentFile, cacheMetaFile, res, reqPath, token, sign, uniqidhex);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||||
@@ -177,7 +203,7 @@ async function tryServeFromStaleCacheOrError(uniqidhex, res, errorMessage) {
|
|||||||
console.warn(`API call failed or returned non-200. Serving stale cache for ${uniqidhex}`);
|
console.warn(`API call failed or returned non-200. Serving stale cache for ${uniqidhex}`);
|
||||||
try {
|
try {
|
||||||
const cacheData = JSON.parse(fs.readFileSync(cacheMetaFile, 'utf8'));
|
const cacheData = JSON.parse(fs.readFileSync(cacheMetaFile, 'utf8'));
|
||||||
serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res);
|
serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res, null, null, null, uniqidhex);
|
||||||
return;
|
return;
|
||||||
} catch (parseError) {
|
} catch (parseError) {
|
||||||
console.error(`Error parsing stale meta file ${cacheMetaFile}:`, parseError);
|
console.error(`Error parsing stale meta file ${cacheMetaFile}:`, parseError);
|
||||||
@@ -225,7 +251,7 @@ async function handleMainRequest(req, res) {
|
|||||||
res.end();
|
res.end();
|
||||||
} else {
|
} else {
|
||||||
viewsInfo.increment('cacheHit');
|
viewsInfo.increment('cacheHit');
|
||||||
serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res);
|
serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res, reqPath, token, sign, uniqidhex);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
try {
|
try {
|
||||||
@@ -255,7 +281,7 @@ const server = http.createServer(handleMainRequest);
|
|||||||
// 检查缓存头并返回是否为304
|
// 检查缓存头并返回是否为304
|
||||||
async function checkCacheHeaders(req, cacheMetaFile) {
|
async function checkCacheHeaders(req, cacheMetaFile) {
|
||||||
try {
|
try {
|
||||||
const metaContent = fs.readFileSync(cacheMetaFile, 'utf8');
|
const metaContent = await fs.promises.readFile(cacheMetaFile, 'utf8');
|
||||||
const cacheData = JSON.parse(metaContent);
|
const cacheData = JSON.parse(metaContent);
|
||||||
const ifNoneMatch = req.headers['if-none-match'];
|
const ifNoneMatch = req.headers['if-none-match'];
|
||||||
const ifModifiedSince = req.headers['if-modified-since'];
|
const ifModifiedSince = req.headers['if-modified-since'];
|
||||||
@@ -283,22 +309,24 @@ async function checkCacheHeaders(req, cacheMetaFile) {
|
|||||||
return { cacheData, isNotModified: false };
|
return { cacheData, isNotModified: false };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error reading or parsing cache meta file ${cacheMetaFile} in checkCacheHeaders:`, error);
|
console.error(`Error reading or parsing cache meta file ${cacheMetaFile} in checkCacheHeaders:`, error);
|
||||||
// If we can't read meta, assume cache is invalid or treat as not modified: false
|
|
||||||
// Returning a dummy cacheData or null might be better depending on how caller handles it.
|
|
||||||
// For now, let it propagate and potentially fail later if cacheData is expected.
|
|
||||||
// Or, more safely, indicate cache is not valid / not modified is false.
|
|
||||||
return { cacheData: null, isNotModified: false }; // Indicate failure to load cacheData
|
return { cacheData: null, isNotModified: false }; // Indicate failure to load cacheData
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// 检查缓存是否有效
|
// 检查缓存是否有效
|
||||||
function isCacheValid(cacheMetaFile, cacheContentFile) {
|
async function isCacheValid(cacheMetaFile, cacheContentFile) {
|
||||||
if (!fs.existsSync(cacheMetaFile) || !fs.existsSync(cacheContentFile)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
const metaContent = fs.readFileSync(cacheMetaFile, 'utf8');
|
// 使用Promise.all并行检查文件是否存在
|
||||||
|
const [metaExists, contentExists] = await Promise.all([
|
||||||
|
fs.promises.access(cacheMetaFile).then(() => true).catch(() => false),
|
||||||
|
fs.promises.access(cacheContentFile).then(() => true).catch(() => false)
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!metaExists || !contentExists) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const metaContent = await fs.promises.readFile(cacheMetaFile, 'utf8');
|
||||||
const cacheData = JSON.parse(metaContent);
|
const cacheData = JSON.parse(metaContent);
|
||||||
// Ensure expiration is a number and in the future
|
// Ensure expiration is a number and in the future
|
||||||
return typeof cacheData.expiration === 'number' && cacheData.expiration > Date.now();
|
return typeof cacheData.expiration === 'number' && cacheData.expiration > Date.now();
|
||||||
@@ -375,51 +403,34 @@ async function fetchApiData(reqPath, token, sign) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// createThumbnail
|
|
||||||
function createThumbnail(data, cacheContentFile) {
|
|
||||||
const { path, thumb } = data;
|
|
||||||
|
|
||||||
const thumbCacheFile = pathModule.join(cacheDir, `thumb_${thumb.uniqid}.jpeg`);
|
|
||||||
if (fs.existsSync(thumbCacheFile)) return thumbCacheFile;
|
|
||||||
|
|
||||||
const isVideo = path && typeof path === 'string' && path.includes('.mp4');
|
|
||||||
if (isVideo || !thumb) return;
|
|
||||||
const width = thumb.width && thumb.width > 0 ? thumb.width : undefined;
|
|
||||||
const height = thumb.height && thumb.height > 0 ? thumb.height : undefined;
|
|
||||||
if (!width) return;
|
|
||||||
sharp(cacheContentFile).resize(width, height).toFile(thumbCacheFile);
|
|
||||||
return thumbCacheFile;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// 从真实 URL 获取数据并写入缓存
|
// 从真实 URL 获取数据并写入缓存
|
||||||
const REAL_URL_FETCH_TIMEOUT_MS = 0; // 0 means no timeout for the actual file download
|
const REAL_URL_FETCH_TIMEOUT_MS = 0; // 0 means no timeout for the actual file download
|
||||||
|
|
||||||
const fetchAndServe = (data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res) => {
|
const fetchAndServe = async (data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res) => {
|
||||||
const protocol = data.realUrl.startsWith('https:') ? https : http;
|
const protocol = data.realUrl.startsWith('https:') ? https : http;
|
||||||
|
|
||||||
protocol.get(data.realUrl, { timeout: REAL_URL_FETCH_TIMEOUT_MS, rejectUnauthorized: false }, (realRes) => {
|
protocol.get(data.realUrl, { timeout: REAL_URL_FETCH_TIMEOUT_MS, rejectUnauthorized: false }, (realRes) => {
|
||||||
const cacheStream = fs.createWriteStream(tempCacheContentFile, { flags: 'w' });
|
const cacheStream = fs.createWriteStream(tempCacheContentFile, { flags: 'w', highWaterMark: 64 * 1024 }); // 增加缓冲区大小到64KB
|
||||||
|
|
||||||
let isVideo = data.path && typeof data.path === 'string' && data.path.includes('.mp4');
|
let isVideo = data.path && typeof data.path === 'string' && data.path.includes('.mp4');
|
||||||
// 确保 content-length 是有效的
|
// 确保 content-length 是有效的
|
||||||
const contentLength = realRes.headers['content-length'];
|
const contentLength = realRes.headers['content-length'];
|
||||||
if (contentLength) {
|
if (contentLength) {
|
||||||
|
|
||||||
// contentLength 小于 2KB 且与缓存文件大小不一致时,重新获取
|
// contentLength 小于 2KB 且与缓存文件大小不一致时,重新获取
|
||||||
if (contentLength < 2048 && data.headers['content-length'] !== contentLength) {
|
if (contentLength < 2048 && data.headers['content-length'] !== contentLength) {
|
||||||
console.warn('Warning: content-length is different for the response from:', data.realUrl);
|
console.warn('Warning: content-length is different for the response from:', data.realUrl);
|
||||||
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, `Bad Gateway: Content-Length mismatch for ${data.realUrl}`);
|
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, `Bad Gateway: Content-Length mismatch for ${data.realUrl}`);
|
||||||
// Clean up temp file if stream hasn't started or failed early
|
// Clean up temp file if stream hasn't started or failed early
|
||||||
if (fs.existsSync(tempCacheContentFile)) {
|
fs.promises.access(tempCacheContentFile)
|
||||||
fs.unlinkSync(tempCacheContentFile);
|
.then(() => fs.promises.unlink(tempCacheContentFile))
|
||||||
}
|
.catch(() => {}); // 忽略文件不存在的错误
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
data.headers['content-length'] = contentLength;
|
data.headers['content-length'] = contentLength;
|
||||||
// 更新 data 到缓存 cacheMetaFile
|
// 异步更新 data 到缓存 cacheMetaFile
|
||||||
fs.writeFileSync(cacheMetaFile, JSON.stringify(data));
|
fs.promises.writeFile(cacheMetaFile, JSON.stringify(data))
|
||||||
|
.catch(err => console.error(`Error writing meta file ${cacheMetaFile}:`, err));
|
||||||
} else {
|
} else {
|
||||||
console.warn('Warning: content-length is undefined for the response from:', data.realUrl);
|
console.warn('Warning: content-length is undefined for the response from:', data.realUrl);
|
||||||
}
|
}
|
||||||
@@ -432,98 +443,192 @@ const fetchAndServe = (data, tempCacheContentFile, cacheContentFile, cacheMetaFi
|
|||||||
'Expires': new Date(Date.now() + 31536000000).toUTCString(),
|
'Expires': new Date(Date.now() + 31536000000).toUTCString(),
|
||||||
'Accept-Ranges': 'bytes',
|
'Accept-Ranges': 'bytes',
|
||||||
'Connection': 'keep-alive',
|
'Connection': 'keep-alive',
|
||||||
'Date': new Date().toUTCString(), // Should be set by the server, but good for consistency
|
'Date': new Date().toUTCString(),
|
||||||
'Last-Modified': data.headers['last-modified'] || new Date(fs.statSync(cacheMetaFile).mtime).toUTCString(), // Prefer API's Last-Modified if available
|
'Last-Modified': data.headers['last-modified'] || new Date().toUTCString(),
|
||||||
};
|
};
|
||||||
const responseHeaders = {
|
const responseHeaders = {
|
||||||
...baseHeaders,
|
...baseHeaders,
|
||||||
'Content-Type': realRes.headers['content-type'] || (isVideo ? 'video/mp4' : 'application/octet-stream'), // Prefer actual content-type
|
'Content-Type': realRes.headers['content-type'] || (isVideo ? 'video/mp4' : 'application/octet-stream'),
|
||||||
...data.headers, // Allow API to override some headers if necessary
|
...data.headers,
|
||||||
};
|
};
|
||||||
|
|
||||||
res.writeHead(realRes.statusCode, responseHeaders);
|
res.writeHead(realRes.statusCode, responseHeaders);
|
||||||
realRes.pipe(cacheStream);
|
|
||||||
realRes.pipe(res);
|
// 使用管道优化流传输
|
||||||
|
const pipeline = require('stream').pipeline;
|
||||||
realRes.on('end', () => {
|
|
||||||
cacheStream.end(() => { // Ensure stream is fully flushed before renaming
|
// 创建一个流分支,同时写入缓存和响应
|
||||||
if (fs.existsSync(tempCacheContentFile)) {
|
const { PassThrough } = require('stream');
|
||||||
try {
|
const passThrough = new PassThrough();
|
||||||
// Ensure the target directory exists before renaming
|
|
||||||
const targetDir = pathModule.dirname(cacheContentFile);
|
passThrough.pipe(cacheStream);
|
||||||
if (!fs.existsSync(targetDir)) {
|
passThrough.pipe(res);
|
||||||
fs.mkdirSync(targetDir, { recursive: true });
|
|
||||||
}
|
// 使用pipeline处理流错误
|
||||||
fs.renameSync(tempCacheContentFile, cacheContentFile);
|
pipeline(
|
||||||
console.log(`Successfully cached: ${cacheContentFile}`);
|
realRes,
|
||||||
|
passThrough,
|
||||||
// 生成缩略图
|
(err) => {
|
||||||
if (data.thumb) {
|
if (err) {
|
||||||
createThumbnail(data, cacheContentFile);
|
console.error(`Pipeline error for ${data.realUrl}:`, err);
|
||||||
}
|
handleResponseError(res, tempCacheContentFile, data.realUrl);
|
||||||
} catch (renameError) {
|
return;
|
||||||
console.error(`Error renaming temp cache file ${tempCacheContentFile} to ${cacheContentFile}:`, renameError);
|
|
||||||
// If rename fails, try to remove the temp file to avoid clutter
|
|
||||||
try { fs.unlinkSync(tempCacheContentFile); } catch (e) { /* ignore */ }
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// This case might indicate an issue if the stream ended but no temp file was created/found
|
|
||||||
console.warn(`Temp cache file ${tempCacheContentFile} not found after stream end for ${data.realUrl}`);
|
|
||||||
}
|
}
|
||||||
});
|
|
||||||
});
|
// 流处理完成后,重命名临时文件
|
||||||
|
fs.promises.access(tempCacheContentFile)
|
||||||
realRes.on('error', (streamError) => {
|
.then(() => {
|
||||||
console.error(`Error during response stream from ${data.realUrl}:`, streamError);
|
// 确保目标目录存在
|
||||||
cacheStream.end(); // Close the writable stream
|
return fs.promises.mkdir(pathModule.dirname(cacheContentFile), { recursive: true })
|
||||||
handleResponseError(res, tempCacheContentFile, data.realUrl); // tempCacheContentFile might be partially written
|
.then(() => fs.promises.rename(tempCacheContentFile, cacheContentFile))
|
||||||
});
|
.then(() => console.log(`Successfully cached: ${cacheContentFile}`))
|
||||||
|
.catch(renameError => {
|
||||||
|
console.error(`Error renaming temp cache file ${tempCacheContentFile} to ${cacheContentFile}:`, renameError);
|
||||||
|
return fs.promises.unlink(tempCacheContentFile).catch(() => {});
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
console.warn(`Temp cache file ${tempCacheContentFile} not found after stream end for ${data.realUrl}`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
}).on('error', (requestError) => {
|
}).on('error', (requestError) => {
|
||||||
console.error(`Error making GET request to ${data.realUrl}:`, requestError);
|
console.error(`Error making GET request to ${data.realUrl}:`, requestError);
|
||||||
// No cacheStream involved here if the request itself fails before response
|
handleResponseError(res, tempCacheContentFile, data.realUrl);
|
||||||
handleResponseError(res, tempCacheContentFile, data.realUrl); // tempCacheContentFile might not exist or be empty
|
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
// 从缓存中读取数据并返回
|
// 从缓存中读取数据并返回
|
||||||
function serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res) {
|
async function serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res, reqPath, token, sign, uniqidhex) {
|
||||||
if (!cacheData) { // Added check for null cacheData from checkCacheHeaders failure
|
if (!cacheData) { // 缓存数据不可用,尝试重新获取
|
||||||
console.error(`serveFromCache called with null cacheData for ${cacheContentFile}`);
|
console.warn(`Cache metadata unavailable for ${cacheContentFile}, attempting to fetch fresh data`);
|
||||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Cache metadata unavailable.');
|
|
||||||
return;
|
// 如果提供了请求参数,尝试重新获取数据
|
||||||
|
if (reqPath && token) {
|
||||||
|
try {
|
||||||
|
viewsInfo.increment('apiCall');
|
||||||
|
const apiData = await fetchApiData(reqPath, token, sign);
|
||||||
|
|
||||||
|
if (apiData.code === HTTP_STATUS.REDIRECT || apiData.code === 301) {
|
||||||
|
res.writeHead(HTTP_STATUS.REDIRECT, { Location: apiData.data.url });
|
||||||
|
res.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (apiData.code === HTTP_STATUS.OK && apiData.data && apiData.data.url) {
|
||||||
|
const { url: realUrl, cloudtype, expiration, path: apiPath, headers, uniqid, thumb } = apiData.data;
|
||||||
|
const data = { realUrl, cloudtype, expiration: expiration * 1000, path: apiPath, headers, uniqid, thumb };
|
||||||
|
|
||||||
|
// 更新索引
|
||||||
|
pathIndex[uniqidhex] = { uniqid: data.uniqid, timestamp: Date.now() };
|
||||||
|
|
||||||
|
// 写入新的元数据
|
||||||
|
await fs.promises.mkdir(pathModule.dirname(cacheMetaFile), { recursive: true });
|
||||||
|
await fs.promises.writeFile(cacheMetaFile, JSON.stringify(data));
|
||||||
|
|
||||||
|
// 获取并提供新数据
|
||||||
|
const tempCacheContentFile = pathModule.join(cacheDir, `${data.uniqid}_${crypto.randomBytes(16).toString('hex')}.temp`);
|
||||||
|
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
viewsInfo.increment('fetchApiWarning');
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, apiData.message || 'Failed to fetch data from API');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
viewsInfo.increment('fetchApiError');
|
||||||
|
console.error('Error fetching fresh data:', error);
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, `Failed to fetch fresh data: ${error.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// 如果没有提供请求参数,无法重新获取
|
||||||
|
console.error(`serveFromCache called with null cacheData and insufficient request info for ${cacheContentFile}`);
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Cache metadata unavailable and cannot fetch fresh data.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 使用异步方式获取ETag和Last-Modified
|
||||||
|
let etag = cacheData.uniqid;
|
||||||
|
let lastModified = cacheData.headers && cacheData.headers['last-modified'];
|
||||||
|
|
||||||
|
if (!etag || !lastModified) {
|
||||||
|
try {
|
||||||
|
const [fileStats, metaStats] = await Promise.all([
|
||||||
|
fs.promises.stat(cacheContentFile).catch(() => null),
|
||||||
|
fs.promises.stat(cacheMetaFile).catch(() => null)
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!etag && fileStats) {
|
||||||
|
// 使用文件大小和修改时间作为ETag的一部分,避免读取整个文件计算MD5
|
||||||
|
etag = crypto.createHash('md5')
|
||||||
|
.update(`${fileStats.size}-${fileStats.mtime.getTime()}`)
|
||||||
|
.digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!lastModified && metaStats) {
|
||||||
|
lastModified = new Date(metaStats.mtime).toUTCString();
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(`Error getting file stats for cache: ${error.message}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const baseHeaders = {
|
const baseHeaders = {
|
||||||
'Cloud-Type': cacheData.cloudtype || 'unknown',
|
'Cloud-Type': cacheData.cloudtype || 'unknown',
|
||||||
'Cloud-Expiration': cacheData.expiration || 'N/A',
|
'Cloud-Expiration': cacheData.expiration || 'N/A',
|
||||||
'ETag': cacheData.uniqid || crypto.createHash('md5').update(fs.readFileSync(cacheContentFile)).digest('hex'), // Fallback ETag if missing
|
'ETag': etag || '',
|
||||||
'Cache-Control': 'public, max-age=31536000', // 1 year
|
'Cache-Control': 'public, max-age=31536000', // 1 year
|
||||||
'Expires': new Date(Date.now() + 31536000000).toUTCString(),
|
'Expires': new Date(Date.now() + 31536000000).toUTCString(),
|
||||||
'Accept-Ranges': 'bytes',
|
'Accept-Ranges': 'bytes',
|
||||||
'Connection': 'keep-alive',
|
'Connection': 'keep-alive',
|
||||||
'Date': new Date().toUTCString(),
|
'Date': new Date().toUTCString(),
|
||||||
'Last-Modified': (cacheData.headers && cacheData.headers['last-modified']) || new Date(fs.statSync(cacheMetaFile).mtime).toUTCString(),
|
'Last-Modified': lastModified || new Date().toUTCString(),
|
||||||
};
|
};
|
||||||
if (cacheData.thumb) {
|
|
||||||
var thumbCacheFile = createThumbnail(cacheData, cacheContentFile)
|
viewsInfo.increment('cacheCall');
|
||||||
if (thumbCacheFile && fs.existsSync(thumbCacheFile)) {
|
|
||||||
cacheData.headers['content-length'] = fs.statSync(thumbCacheFile).size;
|
// 先检查缓存文件是否存在且可读
|
||||||
const responseHeaders = {
|
try {
|
||||||
...baseHeaders,
|
await fs.promises.access(cacheContentFile, fs.constants.R_OK);
|
||||||
...(cacheData.headers || {}),
|
} catch (error) {
|
||||||
'ETag': (cacheData.thumb.uniqid || cacheData.uniqid) + '_thumb',
|
console.warn(`Cache content file ${cacheContentFile} not accessible: ${error.message}`);
|
||||||
'Content-Type': 'image/jpeg',
|
|
||||||
};
|
// 如果提供了请求参数,尝试重新获取数据
|
||||||
res.writeHead(HTTP_STATUS.OK, responseHeaders);
|
if (reqPath && token) {
|
||||||
const thumbStream = fs.createReadStream(thumbCacheFile);
|
console.log(`Attempting to fetch fresh data for ${cacheContentFile}`);
|
||||||
thumbStream.pipe(res);
|
try {
|
||||||
|
viewsInfo.increment('apiCall');
|
||||||
|
const apiData = await fetchApiData(reqPath, token, sign);
|
||||||
|
|
||||||
|
if (apiData.code === HTTP_STATUS.OK && apiData.data && apiData.data.url) {
|
||||||
|
const { url: realUrl, cloudtype, expiration, path: apiPath, headers, uniqid, thumb } = apiData.data;
|
||||||
|
const data = { realUrl, cloudtype, expiration: expiration * 1000, path: apiPath, headers, uniqid, thumb };
|
||||||
|
|
||||||
|
// 更新索引
|
||||||
|
pathIndex[uniqidhex] = { uniqid: data.uniqid, timestamp: Date.now() };
|
||||||
|
|
||||||
|
// 获取并提供新数据
|
||||||
|
const tempCacheContentFile = pathModule.join(cacheDir, `${data.uniqid}_${crypto.randomBytes(16).toString('hex')}.temp`);
|
||||||
|
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, apiData.message || 'Failed to fetch data from API');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (fetchError) {
|
||||||
|
console.error(`Error fetching fresh data: ${fetchError.message}`);
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, `Failed to fetch fresh data: ${fetchError.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Unable to read cache content file and cannot fetch fresh data.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
viewsInfo.increment('cacheCall');
|
const readStream = fs.createReadStream(cacheContentFile, { highWaterMark: 64 * 1024 }); // 增加读取缓冲区大小
|
||||||
const readStream = fs.createReadStream(cacheContentFile);
|
|
||||||
const isVideo = cacheData.path && typeof cacheData.path === 'string' && cacheData.path.includes('.mp4');
|
const isVideo = cacheData.path && typeof cacheData.path === 'string' && cacheData.path.includes('.mp4');
|
||||||
|
|
||||||
let currentContentLength = cacheData.headers && cacheData.headers['content-length'] ? parseInt(cacheData.headers['content-length'], 10) : 0;
|
let currentContentLength = cacheData.headers && cacheData.headers['content-length'] ? parseInt(cacheData.headers['content-length'], 10) : 0;
|
||||||
@@ -544,7 +649,7 @@ function serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res) {
|
|||||||
}
|
}
|
||||||
} catch (statError) {
|
} catch (statError) {
|
||||||
console.error(`Error stating cache content file ${cacheContentFile}:`, statError);
|
console.error(`Error stating cache content file ${cacheContentFile}:`, statError);
|
||||||
handleCacheReadError(res, cacheContentFile); // Treat stat error as read error
|
handleCacheReadError(res, cacheContentFile, reqPath, token, sign, uniqidhex); // Treat stat error as read error
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -565,7 +670,38 @@ function serveFromCache(cacheData, cacheContentFile, cacheMetaFile, res) {
|
|||||||
|
|
||||||
readStream.on('error', (err) => {
|
readStream.on('error', (err) => {
|
||||||
console.error(`Read stream error for ${cacheContentFile}:`, err);
|
console.error(`Read stream error for ${cacheContentFile}:`, err);
|
||||||
handleCacheReadError(res, cacheContentFile);
|
|
||||||
|
// 如果提供了请求参数,尝试重新获取数据而不是直接报错
|
||||||
|
if (reqPath && token) {
|
||||||
|
console.log(`Read stream error, attempting to fetch fresh data for ${cacheContentFile}`);
|
||||||
|
viewsInfo.increment('apiCall');
|
||||||
|
|
||||||
|
fetchApiData(reqPath, token, sign)
|
||||||
|
.then(apiData => {
|
||||||
|
if (apiData.code === HTTP_STATUS.OK && apiData.data && apiData.data.url) {
|
||||||
|
const { url: realUrl, cloudtype, expiration, path: apiPath, headers, uniqid, thumb } = apiData.data;
|
||||||
|
const data = { realUrl, cloudtype, expiration: expiration * 1000, path: apiPath, headers, uniqid, thumb };
|
||||||
|
|
||||||
|
// 更新索引
|
||||||
|
pathIndex[uniqidhex] = { uniqid: data.uniqid, timestamp: Date.now() };
|
||||||
|
|
||||||
|
// 获取并提供新数据
|
||||||
|
const tempCacheContentFile = pathModule.join(cacheDir, `${data.uniqid}_${crypto.randomBytes(16).toString('hex')}.temp`);
|
||||||
|
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||||
|
} else {
|
||||||
|
viewsInfo.increment('fetchApiWarning');
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, apiData.message || 'Failed to fetch data from API');
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(fetchError => {
|
||||||
|
viewsInfo.increment('fetchApiError');
|
||||||
|
console.error(`Error fetching fresh data after read stream error: ${fetchError.message}`);
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, `Failed to fetch fresh data: ${fetchError.message}`);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// 如果没有提供请求参数,使用原始的错误处理
|
||||||
|
handleCacheReadError(res, cacheContentFile, reqPath, token, sign, uniqidhex);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Handle cases where client closes connection prematurely
|
// Handle cases where client closes connection prematurely
|
||||||
@@ -593,10 +729,55 @@ const handleResponseError = (res, tempCacheContentFile, realUrl) => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// 处理缓存读取错误
|
// 处理缓存读取错误
|
||||||
const handleCacheReadError = (res, filePath) => {
|
const handleCacheReadError = (res, filePath, reqPath, token, sign, uniqidhex) => {
|
||||||
viewsInfo.increment('cacheReadError');
|
viewsInfo.increment('cacheReadError');
|
||||||
console.error(`Error reading cache file: ${filePath}`);
|
console.error(`Error reading cache file: ${filePath}`);
|
||||||
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Internal Server Error: Unable to read cache content file');
|
|
||||||
|
// 如果提供了请求参数,尝试重新获取数据
|
||||||
|
if (reqPath && token) {
|
||||||
|
console.log(`Cache read error, attempting to fetch fresh data for ${filePath}`);
|
||||||
|
viewsInfo.increment('apiCall');
|
||||||
|
|
||||||
|
fetchApiData(reqPath, token, sign)
|
||||||
|
.then(apiData => {
|
||||||
|
if (apiData.code === HTTP_STATUS.OK && apiData.data && apiData.data.url) {
|
||||||
|
const { url: realUrl, cloudtype, expiration, path: apiPath, headers, uniqid, thumb } = apiData.data;
|
||||||
|
const data = { realUrl, cloudtype, expiration: expiration * 1000, path: apiPath, headers, uniqid, thumb };
|
||||||
|
|
||||||
|
// 更新索引
|
||||||
|
if (uniqidhex) {
|
||||||
|
pathIndex[uniqidhex] = { uniqid: data.uniqid, timestamp: Date.now() };
|
||||||
|
}
|
||||||
|
|
||||||
|
// 获取并提供新数据
|
||||||
|
const cacheMetaFile = pathModule.join(cacheDir, `${uniqidhex}.meta`);
|
||||||
|
const cacheContentFile = pathModule.join(cacheDir, `${data.uniqid}.content`);
|
||||||
|
const tempCacheContentFile = pathModule.join(cacheDir, `${data.uniqid}_${crypto.randomBytes(16).toString('hex')}.temp`);
|
||||||
|
|
||||||
|
// 写入新的元数据
|
||||||
|
fs.promises.mkdir(pathModule.dirname(cacheMetaFile), { recursive: true })
|
||||||
|
.then(() => fs.promises.writeFile(cacheMetaFile, JSON.stringify(data)))
|
||||||
|
.then(() => {
|
||||||
|
fetchAndServe(data, tempCacheContentFile, cacheContentFile, cacheMetaFile, res);
|
||||||
|
})
|
||||||
|
.catch(writeError => {
|
||||||
|
console.error(`Error writing meta file after cache read error: ${writeError.message}`);
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Failed to write cache metadata');
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
viewsInfo.increment('fetchApiWarning');
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.BAD_GATEWAY, apiData.message || 'Failed to fetch data from API');
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(fetchError => {
|
||||||
|
viewsInfo.increment('fetchApiError');
|
||||||
|
console.error(`Error fetching fresh data after cache read error: ${fetchError.message}`);
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, `Failed to fetch fresh data: ${fetchError.message}`);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
// 如果没有提供请求参数,返回错误
|
||||||
|
sendErrorResponse(res, HTTP_STATUS.INTERNAL_SERVER_ERROR, 'Internal Server Error: Unable to read cache content file');
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// 启动服务器
|
// 启动服务器
|
||||||
|
|||||||
Reference in New Issue
Block a user