232 lines
8.1 KiB
JavaScript
232 lines
8.1 KiB
JavaScript
#!/usr/bin/env node
|
|
'use strict';
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const https = require('https');
|
|
|
|
const ROOT = __dirname;
|
|
const SOURCE_FILE = path.join(ROOT, 'catalog', 'photo-sources.json');
|
|
const EXPECTED_TOTAL = 113;
|
|
const MAX_BYTES = 20 * 1024 * 1024;
|
|
const CONCURRENCY = 3;
|
|
const RETRIES = 3;
|
|
const CHECK_ONLY = process.argv.includes('--check-only');
|
|
const FORCE = process.argv.includes('--force');
|
|
|
|
function normalizeRel(value) {
|
|
return String(value || '').replace(/\\/g, '/').replace(/^\/+/, '');
|
|
}
|
|
|
|
function readSources() {
|
|
let data;
|
|
try {
|
|
data = JSON.parse(fs.readFileSync(SOURCE_FILE, 'utf8'));
|
|
} catch (error) {
|
|
throw new Error(`Cannot read catalog/photo-sources.json: ${error.message}`);
|
|
}
|
|
if (!Array.isArray(data)) throw new Error('catalog/photo-sources.json must contain an array');
|
|
const out = data.map((item, index) => ({
|
|
index: index + 1,
|
|
path: normalizeRel(item && item.path),
|
|
url: String(item && item.url || '').trim()
|
|
})).filter(item => item.path && item.url);
|
|
if (out.length !== EXPECTED_TOTAL) {
|
|
throw new Error(`Expected ${EXPECTED_TOTAL} catalog photo sources, found ${out.length}`);
|
|
}
|
|
const uniquePaths = new Set(out.map(item => item.path));
|
|
if (uniquePaths.size !== EXPECTED_TOTAL) throw new Error('Catalog photo source paths are not unique');
|
|
return out;
|
|
}
|
|
|
|
function isAllowedUrl(target) {
|
|
let url;
|
|
try { url = new URL(target); } catch (_) { return false; }
|
|
if (url.protocol !== 'https:') return false;
|
|
return new Set(['static.tildacdn.com', 'static3.tildacdn.com', 'static.tildacdn.net']).has(url.hostname);
|
|
}
|
|
|
|
function imageKind(buf) {
|
|
if (!Buffer.isBuffer(buf) || buf.length < 12) return '';
|
|
if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return 'jpeg';
|
|
if (buf.subarray(0, 8).equals(Buffer.from([0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a]))) return 'png';
|
|
if (buf.toString('ascii', 0, 4) === 'RIFF' && buf.toString('ascii', 8, 12) === 'WEBP') return 'webp';
|
|
return '';
|
|
}
|
|
|
|
function validLocalPhoto(rel) {
|
|
const file = path.resolve(ROOT, rel);
|
|
if (!file.startsWith(ROOT + path.sep)) return false;
|
|
try {
|
|
const st = fs.statSync(file);
|
|
if (!st.isFile() || st.size < 1000) return false;
|
|
const fd = fs.openSync(file, 'r');
|
|
const head = Buffer.alloc(16);
|
|
fs.readSync(fd, head, 0, head.length, 0);
|
|
fs.closeSync(fd);
|
|
return imageKind(head) === 'jpeg';
|
|
} catch (_) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function delay(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
function downloadBuffer(target, redirects = 0, timeoutMs = 25000) {
|
|
return new Promise((resolve, reject) => {
|
|
if (redirects > 6) return reject(new Error('too many redirects'));
|
|
let url;
|
|
try { url = new URL(target); } catch (error) { return reject(error); }
|
|
if (!isAllowedUrl(url.toString())) return reject(new Error(`photo host is not allowed: ${url.hostname}`));
|
|
|
|
const req = https.get(url, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/142 Safari/537.36 SunCateringCatalogPhotoPrep/17.5.17',
|
|
'Accept': 'image/jpeg,image/*;q=0.8,*/*;q=0.1',
|
|
'Accept-Language': 'ru-RU,ru;q=0.9,en;q=0.5',
|
|
'Accept-Encoding': 'identity',
|
|
'Referer': 'https://solnce-keytering.ru/'
|
|
},
|
|
timeout: timeoutMs
|
|
}, res => {
|
|
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
res.resume();
|
|
const next = new URL(res.headers.location, url).toString();
|
|
if (!isAllowedUrl(next)) return reject(new Error('redirected to an unsupported photo host'));
|
|
return downloadBuffer(next, redirects + 1, timeoutMs).then(resolve, reject);
|
|
}
|
|
if (res.statusCode < 200 || res.statusCode >= 300) {
|
|
res.resume();
|
|
return reject(new Error(`HTTP ${res.statusCode}`));
|
|
}
|
|
const type = String(res.headers['content-type'] || '').toLowerCase();
|
|
if (type && !type.startsWith('image/')) {
|
|
res.resume();
|
|
return reject(new Error(`unexpected content type: ${type}`));
|
|
}
|
|
const chunks = [];
|
|
let size = 0;
|
|
res.on('data', chunk => {
|
|
size += chunk.length;
|
|
if (size > MAX_BYTES) {
|
|
req.destroy(new Error('photo is too large'));
|
|
return;
|
|
}
|
|
chunks.push(chunk);
|
|
});
|
|
res.on('end', () => {
|
|
const buf = Buffer.concat(chunks);
|
|
const kind = imageKind(buf);
|
|
if (kind !== 'jpeg' || buf.length < 1000) return reject(new Error('downloaded file is not a valid JPEG'));
|
|
resolve(buf);
|
|
});
|
|
res.on('error', reject);
|
|
});
|
|
req.on('timeout', () => req.destroy(new Error('timeout')));
|
|
req.on('error', reject);
|
|
});
|
|
}
|
|
|
|
function atomicWrite(rel, buffer) {
|
|
const file = path.resolve(ROOT, rel);
|
|
if (!file.startsWith(ROOT + path.sep)) throw new Error('invalid local photo path');
|
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
const tmp = `${file}.download-${process.pid}-${Date.now()}`;
|
|
fs.writeFileSync(tmp, buffer);
|
|
fs.renameSync(tmp, file);
|
|
}
|
|
|
|
async function fetchWithRetry(url) {
|
|
let last;
|
|
for (let attempt = 1; attempt <= RETRIES; attempt++) {
|
|
try { return await downloadBuffer(url); }
|
|
catch (error) {
|
|
last = error;
|
|
if (attempt < RETRIES) await delay(700 * attempt);
|
|
}
|
|
}
|
|
throw last || new Error('download failed');
|
|
}
|
|
|
|
function summarize(sources) {
|
|
const ready = sources.filter(item => validLocalPhoto(item.path));
|
|
const missing = sources.filter(item => !validLocalPhoto(item.path));
|
|
return { ready, missing };
|
|
}
|
|
|
|
async function main() {
|
|
const sources = readSources();
|
|
const initial = summarize(sources);
|
|
console.log(`Catalog photos: ${initial.ready.length}/${sources.length} local.`);
|
|
|
|
if (CHECK_ONLY) {
|
|
if (initial.missing.length) {
|
|
console.log(`Missing: ${initial.missing.length}.`);
|
|
process.exitCode = 2;
|
|
} else {
|
|
console.log('All catalog photos are local and valid.');
|
|
}
|
|
return;
|
|
}
|
|
|
|
const targets = FORCE ? sources : initial.missing;
|
|
if (!targets.length) {
|
|
console.log('All 113 catalog photos are already local. No network access is needed.');
|
|
return;
|
|
}
|
|
|
|
const byUrl = new Map();
|
|
for (const item of targets) {
|
|
if (!isAllowedUrl(item.url)) throw new Error(`Unsupported photo URL for ${item.path}`);
|
|
if (!byUrl.has(item.url)) byUrl.set(item.url, []);
|
|
byUrl.get(item.url).push(item);
|
|
}
|
|
const jobs = [...byUrl.entries()].map(([url, items]) => ({ url, items }));
|
|
console.log(`Preparing ${targets.length} missing file(s) from ${jobs.length} unique image(s).`);
|
|
console.log('This happens only once. After completion the app uses local catalog photos.');
|
|
|
|
let cursor = 0;
|
|
let finished = 0;
|
|
const failures = [];
|
|
async function worker() {
|
|
while (true) {
|
|
const pos = cursor++;
|
|
if (pos >= jobs.length) return;
|
|
const job = jobs[pos];
|
|
try {
|
|
const buf = await fetchWithRetry(job.url);
|
|
for (const item of job.items) atomicWrite(item.path, buf);
|
|
finished += job.items.length;
|
|
const current = initial.ready.length + finished;
|
|
console.log(`[${current}/${sources.length}] ${job.items.map(x => path.basename(x.path)).join(', ')}`);
|
|
} catch (error) {
|
|
failures.push({ job, error });
|
|
console.error(`FAILED: ${job.items.map(x => x.path).join(', ')} - ${error.message}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, jobs.length) }, () => worker()));
|
|
|
|
const final = summarize(sources);
|
|
if (final.missing.length) {
|
|
console.error('');
|
|
console.error(`Catalog photo preparation is incomplete: ${final.ready.length}/${sources.length} ready.`);
|
|
console.error('Check the internet connection and run this file again. Existing downloaded photos are kept.');
|
|
console.error('Missing files:');
|
|
console.error(final.missing.map(x => path.basename(x.path)).join(', '));
|
|
process.exitCode = 2;
|
|
return;
|
|
}
|
|
|
|
console.log('');
|
|
console.log('Catalog photos ready: 113/113. The app can now use the catalog photos locally.');
|
|
}
|
|
|
|
main().catch(error => {
|
|
console.error('Catalog photo preparation error:', error && error.message ? error.message : String(error));
|
|
process.exitCode = 2;
|
|
});
|