diff --git a/README.md b/README.md index 711945c..275ca7b 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ npm run dev # 本地预览 } ``` +可选字段（按需填写）： + +- `extraOrigins`：图片放在独立图床时，把图床域名加进 `img-src` 白名单，如 `["https://image.example.com:443"]`。 +- `alwaysFetch`：sitemap 未收录但希望始终刷新的页面，如 `["/links"]`。 +- `extraAssets`：希望镜像到本地的外站资源（如动态壁纸视频），`url` 为原地址、`path` 为镜像内相对路径；已存在时跳过，随 CI 缓存保留。 + 提交 PR 即视为同意本站对博客做静态镜像（保留署名、功能引导回原站）。CI 会自动校验格式。 ## 部署 diff --git a/scripts/mirror.mjs b/scripts/mirror.mjs index 0e2fac7..3cd7c53 100644 --- a/scripts/mirror.mjs +++ b/scripts/mirror.mjs @@ -78,6 +78,8 @@ async function processChannel(channel) { } } + await fetchExtraAssets(channel, hostDir); + try { await sanitizeChannel({ hostDir, @@ -86,6 +88,8 @@ async function processChannel(channel) { origin: channel.url, prefix, base: base || '', + extraOrigins: channel.extraOrigins || [], + extraAssets: channel.extraAssets || [], }); const { sizeBytes, fileCount } = await dirStats(outDir); entry.sizeBytes = sizeBytes; @@ -102,6 +106,31 @@ async function processChannel(channel) { return entry; } +// 把配置里声明的外站资源（如动态壁纸视频）镜像到本地，避免访客每次访问都请求友站图床。 +// 只下载一次：文件已存在则跳过，随 Actions 缓存保留。 +async function fetchExtraAssets(channel, hostDir) { + if (!hostDir) return; + for (const asset of channel.extraAssets || []) { + if (!asset?.url || !asset?.path || asset.path.includes('..')) continue; + const target = path.join(hostDir, asset.path); + if (existsSync(target)) continue; + await mkdir(path.dirname(target), { recursive: true }); + console.log(`[${channel.id}] 下载额外资源 ${asset.url}`); + try { + const res = await fetch(asset.url, { signal: AbortSignal.timeout(120000) }); + if (!res.ok) { + console.warn(`[${channel.id}] 额外资源下载失败 HTTP ${res.status}：${asset.url}`); + continue; + } + const buf = Buffer.from(await res.arrayBuffer()); + await writeFile(target, buf); + console.log(`[${channel.id}] 额外资源已保存 ${asset.path}（${(buf.length / 1048576).toFixed(1)} MB）`); + } catch (e) { + console.warn(`[${channel.id}] 额外资源下载失败：${asset.url} (${e.message})`); + } + } +} + // 增量更新：对照 sitemap + 状态文件，只拉新增/更新的页面，并删除源站已移除的页面 async function tryIncremental(channel, cacheDir) { const stateFile = path.join(stateRoot, `${channel.id}.json`); @@ -149,7 +178,9 @@ async function tryIncremental(channel, cacheDir) { const origin = new URL(channel.url).origin; const urls = new Set(toFetch.map((p) => origin + p)); - for (const p of ALWAYS_FETCH) urls.add(origin + p); + const always = new Set(ALWAYS_FETCH); + for (const p of channel.alwaysFetch || []) always.add(p); + for (const p of always) urls.add(origin + p); let fetched = 0; if (urls.size) { @@ -158,7 +189,10 @@ async function tryIncremental(channel, cacheDir) { '-E', '-k', '-p', '-np', '-N', '-nv', '--timeout=30', '--tries=3', '--reject-regex', '\\?width=', '-P', cacheDir, ...urls, ]; - spawnSync('wget', args, { stdio: 'inherit' }); + const res = spawnSync('wget', args, { stdio: 'inherit' }); + if (res.status !== 0) { + console.warn(`[${channel.id}] 增量抓取 wget 退出码 ${res.status}，沿用本地已有缓存`); + } } // 删除源站 sitemap 中已消失的页面 diff --git a/scripts/sanitize.mjs b/scripts/sanitize.mjs index a63d57b..afb9b06 100644 --- a/scripts/sanitize.mjs +++ b/scripts/sanitize.mjs @@ -1,21 +1,44 @@ import { readFile, writeFile, mkdir, rm, copyFile, readdir, open } from 'node:fs/promises'; import { existsSync } from 'node:fs'; import path from 'node:path'; +import { gunzipSync } from 'node:zlib'; import { load } from 'cheerio'; const REMOVE_SCRIPT_RE = /(halo-tracker|PluginCommentWidget|PluginSearchWidget|metmusic|summaraid|analytics|umami|plausible|goatcounter|matomo|tracker)/i; const INLINE_REMOVE_RE = /\/plugins\/(PluginCommentWidget|PluginSearchWidget)/; -const KNOWN_ROOT_RE = /^(["'])\/(upload|themes|plugins|archives|tags|categories|links|pages|images|search|rss|sitemap|favicon|about-me|met-website)\//; -const CSP = - "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' data:; media-src 'self'; connect-src 'none'; frame-src 'none'; object-src 'none'; base-uri 'self'; form-action 'self'"; +const KNOWN_ROOT_RE = + /(["'])\/(upload|themes|plugins|archives|tags|categories|links|pages|images|search|rss|sitemap|favicon|about-me|met-website)\/([^"'\s\\]*)/g; +// 部分友站会把图片/头像放在独立图床，镜像不抓跨域资源， +// 需要把这些来源加进 img-src 白名单，页面才能正常显示图片。 +function buildCsp(extraOrigins = []) { + const img = ["'self'", "'data:'", ...extraOrigins].join(' '); + const media = ["'self'", ...extraOrigins].join(' '); + return `default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src ${img}; font-src 'self' data:; media-src ${media}; connect-src 'none'; frame-src 'none'; object-src 'none'; base-uri 'self'; form-action 'self'`; +} + +function escapeRegExp(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} -function rewriteRoot(value, prefix) { +function rewriteRoot(value, prefix, origin) { if (typeof value !== 'string') return value; const clean = value.replace(/\?width=[^&]*/, ''); if (clean.startsWith('/') && !clean.startsWith('//')) { return prefix + clean; } + // 同源绝对链接（http(s)://原站/... 或 //原站/...）也改写为镜像本地路径， + // 否则会被 CSP img-src 'self' 拦截成破图。 + if (origin && /^(https?:)?\/\//i.test(clean)) { + try { + const u = new URL(clean, origin); + if (u.origin === new URL(origin).origin) { + return prefix + u.pathname + (u.search || ''); + } + } catch { + // 解析失败则原样保留 + } + } return clean; } @@ -26,9 +49,9 @@ function rewriteAssetExt(value) { : value; } -function cleanRef(value, prefix) { +function cleanRef(value, prefix, origin) { if (typeof value !== 'string') return value; - let v = rewriteAssetExt(rewriteRoot(value, prefix)); + let v = rewriteAssetExt(rewriteRoot(value, prefix, origin)); // wget 把含查询串的资源存成字面 "?":main.css?v=1.0.6，引用需编码为 %3F， // 否则浏览器把 ? 当查询串、按 main.css 找文件 → 404（页面白屏） if (!/^(https?:)?\/\//i.test(v)) v = v.replace(/\?/g, '%3F'); @@ -54,13 +77,13 @@ function resolveLocalPath(pathname, outDir) { return null; } -function rewriteSrcset(value, prefix) { +function rewriteSrcset(value, prefix, origin) { return value .split(',') .map((part) => { const m = part.trim().match(/^(\S+)(\s+.+)?$/); if (!m) return part; - return cleanRef(m[1], prefix) + (m[2] ?? ''); + return cleanRef(m[1], prefix, origin) + (m[2] ?? ''); }) .join(', '); } @@ -90,21 +113,68 @@ async function looksLikeHtml(file) { } } -function rewriteJsonStrings(node, prefix) { +function rewriteJsonStrings(node, prefix, origin) { if (typeof node === 'string') { - return cleanRef(node, prefix); + return cleanRef(node, prefix, origin); } if (Array.isArray(node)) { - return node.map((v) => rewriteJsonStrings(v, prefix)); + return node.map((v) => rewriteJsonStrings(v, prefix, origin)); } if (node && typeof node === 'object') { const out = {}; - for (const [k, v] of Object.entries(node)) out[k] = rewriteJsonStrings(v, prefix); + for (const [k, v] of Object.entries(node)) out[k] = rewriteJsonStrings(v, prefix, origin); return out; } return node; } +function rewriteCssUrls(css, prefix, origin) { + // 根相对路径 url(/upload/...) + css = css.replace(/(url\(\s*['"]?)\//g, `$1${prefix}/`); + // 同源绝对路径 url(https://原站/upload/...) + if (origin) { + const o = origin.replace(/\/$/, ''); + css = css.replace(new RegExp(`(url\\(\\s*['"]?)${escapeRegExp(o)}`, 'g'), `$1${prefix}`); + } + // 去掉 .gz 后缀的引用（镜像内只有未压缩版） + css = css.replace(/(url\(\s*['"]?)([^)'"]+)/g, (m, pre, p) => pre + rewriteAssetExt(p)); + return css; +} + +// 部分友站对 CSS/JS 下发的是 gzip 字节（Content-Encoding: gzip）， +// wget 会原样存成 *.css/*.js，浏览器从 GitHub Pages 拿到后无法解析。 +// 这里把缓存里的 gzip 资源解压为明文；.gz 后缀的再落到无后缀文件名。 +async function decompressGzAssets(dir) { + for (const file of await listFiles(dir)) { + if (!/\.gz$/i.test(file)) continue; + try { + const buf = await readFile(file); + const out = gunzipSync(buf); + const target = file.replace(/\.gz$/i, ''); + await writeFile(target, out); + await rm(file, { force: true }); + console.log(`[sanitize] 解压 ${path.relative(dir, file)} -> ${path.relative(dir, target)}`); + } catch (e) { + console.warn(`[sanitize] 跳过无法解压的 ${path.relative(dir, file)}: ${e.message}`); + } + } + for (const file of await listFiles(dir)) { + if (!/\.(css|js|html?|svg|xml|json|txt)(\?.*)?$/i.test(file)) continue; + try { + const fh = await open(file, 'r'); + const magic = Buffer.alloc(2); + const { bytesRead } = await fh.read(magic, 0, 2, 0); + await fh.close(); + if (bytesRead < 2 || magic[0] !== 0x1f || magic[1] !== 0x8b) continue; + const data = await readFile(file); + await writeFile(file, gunzipSync(data)); + console.log(`[sanitize] 解压 ${path.relative(dir, file)}`); + } catch (e) { + console.warn(`[sanitize] 跳过无法解压的 ${path.relative(dir, file)}: ${e.message}`); + } + } +} + async function copyTree(src, dest) { await mkdir(dest, { recursive: true }); for (const entry of await readdir(src, { withFileTypes: true })) { @@ -125,7 +195,7 @@ async function listFiles(dir) { return out; } -async function sanitizeHtml(file, { origin, prefix, base, outDir }) { +async function sanitizeHtml(file, { origin, prefix, base, outDir, extraOrigins = [], assetMap = {} }) { let html = await readFile(file, 'utf8'); const $ = load(html, { decodeEntities: false }); let faviconPath = null; @@ -148,16 +218,25 @@ async function sanitizeHtml(file, { origin, prefix, base, outDir }) { $('[href]').each((_, el) => { const v = $(el).attr('href'); - const nv = cleanRef(v, prefix); + // 链接由下方专门的同源解析逻辑处理，避免把外站/未镜像链接误改成本地 404 + const nv = cleanRef(v, prefix, (el.tagName || '').toLowerCase() === 'a' ? null : origin); if (nv !== v) $(el).attr('href', nv); }); $('[src]').each((_, el) => { const v = $(el).attr('src'); - const nv = cleanRef(v, prefix); + const nv = cleanRef(v, prefix, origin); if (nv !== v) $(el).attr('src', nv); + // 音乐组件等会把专辑封面误写成页面地址（xxx.html）， + // 转成透明占位图，避免满页破图图标 + if ((el.tagName || '').toLowerCase() === 'img' && /\.html?$/i.test(nv || '')) { + $(el).attr( + 'src', + 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7', + ); + } }); $('[srcset]').each((_, el) => { - $(el).attr('srcset', rewriteSrcset($(el).attr('srcset'), prefix)); + $(el).attr('srcset', rewriteSrcset($(el).attr('srcset'), prefix, origin)); }); // 同源绝对链接改写为镜像本地路径（点击后仍在 iframe 内导航）； // 外站或找不到本地文件的链接一律新标签打开，避免跨域导航被 X-Frame-Options @@ -191,27 +270,27 @@ async function sanitizeHtml(file, { origin, prefix, base, outDir }) { }); $('[data-src]').each((_, el) => { const v = $(el).attr('data-src'); - const nv = cleanRef(v, prefix); + const nv = cleanRef(v, prefix, origin); if (nv !== v) $(el).attr('data-src', nv); }); $('[action]').each((_, el) => { const v = $(el).attr('action'); - const nv = rewriteRoot(v, prefix); + const nv = rewriteRoot(v, prefix, origin); if (nv !== v) $(el).attr('action', nv); }); $('meta[property^="og:"], meta[name^="twitter:"]').each((_, el) => { const v = $(el).attr('content'); - const nv = cleanRef(v, prefix); + const nv = cleanRef(v, prefix, origin); if (nv !== v) $(el).attr('content', nv); }); $('[style]').each((_, el) => { - const v = $(el).attr('style'); - const nv = v.replace(/(url\(\s*['"]?)\//g, `$1${prefix}/`); + const v = $(el).attr('style') || ''; + const nv = rewriteCssUrls(v, prefix, origin); if (nv !== v) $(el).attr('style', nv); }); // 注入 CSP 与镜像提示资源（放在属性重写之后，避免被二次加前缀） - $('head').prepend(``); + $('head').prepend(``); $('head').prepend(``); $('head').prepend(``); $('head').append(``); @@ -220,8 +299,8 @@ async function sanitizeHtml(file, { origin, prefix, base, outDir }) { $('script[type="application/json"]').each((_, el) => { const raw = $(el).text(); try { - const data = JSON.parse(raw); - $(el).text(JSON.stringify(rewriteJsonStrings(data, prefix))); + const data = rewriteJsonStrings(JSON.parse(raw), prefix, origin); + $(el).text(JSON.stringify(data)); } catch { // 非 JSON 原样保留 } @@ -237,35 +316,62 @@ async function sanitizeHtml(file, { origin, prefix, base, outDir }) { } html = $.html(); - // 处理内联 JS 中带引号的根绝对路径（themeConfig 等） - html = html.replace( - KNOWN_ROOT_RE, - (m, q, dir, rest) => `${q}${prefix}/${dir}/${rewriteAssetExt(rest)}`, + // 镜像到本地的外站资源（如动态壁纸视频）：把原站地址精确替换为本地副本， + // 保持原页面结构与主题配置不变（type 仍为 video，