jeremy
c2e5c1d9dc
- /photo full-screen slideshow (crossfade, blurred backdrop or fill, shuffle, date) - admin Photos tab: add shared-album links (check, sync now, status/stale warnings), slideshow options, browser-resized uploads - 6-hourly album sync scraping the share page (Library API no longer exposes user albums), caches 1920px + 64px copies in /data/photos - /api/health/photos -> 503 when an album has not synced for 48h - history/restore covers photo settings; min admin password length 6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01E9NnVEico3WvFxLyjy1iW3
368 lines
15 KiB
TypeScript
368 lines
15 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { mkdir, readdir, readFile, rename, rm, unlink, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { type PhotoAlbum, readSettings } from "./store";
|
|
|
|
/*
|
|
* Google Photos shared albums. Google removed third-party read access to user albums from the
|
|
* Library API on 2025-03-31, so this reads the public share page the same way a browser does:
|
|
* the album's items are embedded as JSON in an AF_initDataCallback({key: 'ds:1', ...}) block.
|
|
* This is unofficial and may break when Google changes the page. Cached photos keep showing
|
|
* if it does, and the album reports lastError plus a stale lastSuccessAt.
|
|
*/
|
|
|
|
export const MAX_PHOTOS_PER_ALBUM = 500;
|
|
export const MAX_UPLOADS = 500;
|
|
export const SYNC_INTERVAL_MS = 6 * 60 * 60 * 1000;
|
|
export const STALE_AFTER_MS = 48 * 60 * 60 * 1000;
|
|
const PHOTO_EDGE = 1920;
|
|
const THUMB_EDGE = 64;
|
|
const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36";
|
|
const VIDEO_MARKER = "76647426";
|
|
|
|
export type CachedPhoto = { key: string; takenAt: number | null; width: number; height: number };
|
|
export type AlbumIndex = {
|
|
title: string;
|
|
firstAttemptAt: number;
|
|
lastAttemptAt: number;
|
|
lastSuccessAt: number | null;
|
|
lastError: string;
|
|
truncated: boolean;
|
|
photos: CachedPhoto[];
|
|
};
|
|
|
|
const photosRoot = () => path.join(process.env.ORGANIZER_DATA_DIR || "/data", "photos");
|
|
export const UPLOADS_DIR = "uploads";
|
|
const dirFor = (name: string) => path.join(photosRoot(), name);
|
|
const indexPath = (name: string) => path.join(dirFor(name), "index.json");
|
|
|
|
export function validPhotoDir(name: string) {
|
|
return name === UPLOADS_DIR || /^[A-Za-z0-9-]{8,64}$/.test(name);
|
|
}
|
|
export function validPhotoFile(name: string) {
|
|
return /^[a-f0-9]{20}(\.t)?\.jpg$/.test(name);
|
|
}
|
|
export const photoFilePath = (dir: string, file: string) => path.join(dirFor(dir), file);
|
|
|
|
function emptyIndex(): AlbumIndex {
|
|
return { title: "", firstAttemptAt: 0, lastAttemptAt: 0, lastSuccessAt: null, lastError: "", truncated: false, photos: [] };
|
|
}
|
|
|
|
export async function readIndex(name: string): Promise<AlbumIndex> {
|
|
try {
|
|
return { ...emptyIndex(), ...(JSON.parse(await readFile(indexPath(name), "utf8")) as Partial<AlbumIndex>) };
|
|
} catch {
|
|
return emptyIndex();
|
|
}
|
|
}
|
|
|
|
async function writeIndex(name: string, index: AlbumIndex) {
|
|
await mkdir(dirFor(name), { recursive: true, mode: 0o700 });
|
|
const temporary = `${indexPath(name)}.${process.pid}.tmp`;
|
|
await writeFile(temporary, JSON.stringify(index), { mode: 0o600 });
|
|
await rename(temporary, indexPath(name));
|
|
}
|
|
|
|
async function writeFileAtomic(file: string, data: Uint8Array) {
|
|
const temporary = `${file}.${process.pid}.tmp`;
|
|
await writeFile(temporary, data, { mode: 0o600 });
|
|
await rename(temporary, file);
|
|
}
|
|
|
|
export function isJpeg(data: Uint8Array) {
|
|
return data.length > 4 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff;
|
|
}
|
|
|
|
async function readLimited(response: Response, maxBytes: number) {
|
|
const declared = Number(response.headers.get("content-length") ?? 0);
|
|
if (declared > maxBytes) throw new Error("Response too large");
|
|
if (!response.body) return new Uint8Array();
|
|
const reader = response.body.getReader();
|
|
const chunks: Uint8Array[] = [];
|
|
let total = 0;
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
total += value.byteLength;
|
|
if (total > maxBytes) {
|
|
await reader.cancel();
|
|
throw new Error("Response too large");
|
|
}
|
|
chunks.push(value);
|
|
}
|
|
const out = new Uint8Array(total);
|
|
let offset = 0;
|
|
for (const chunk of chunks) {
|
|
out.set(chunk, offset);
|
|
offset += chunk.byteLength;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// ---------- scraping ----------
|
|
|
|
type ScrapedItem = { googleId: string; baseUrl: string; width: number; height: number; takenAt: number | null; video: boolean };
|
|
export type ScrapedAlbum = { title: string; items: ScrapedItem[]; hasMorePages: boolean };
|
|
|
|
const ALLOWED_PAGE_HOSTS = new Set(["photos.app.goo.gl", "photos.google.com"]);
|
|
|
|
async function fetchAlbumPage(shareUrl: string) {
|
|
let current = shareUrl;
|
|
for (let hop = 0; hop <= 5; hop += 1) {
|
|
const url = new URL(current);
|
|
if (url.protocol !== "https:" || !ALLOWED_PAGE_HOSTS.has(url.hostname)) throw new Error(`Album link redirected to an unexpected host (${url.hostname})`);
|
|
const response = await fetch(url, {
|
|
headers: { "User-Agent": USER_AGENT, "Accept-Language": "en-US,en;q=0.9", Accept: "text/html" },
|
|
redirect: "manual",
|
|
signal: AbortSignal.timeout(20_000),
|
|
});
|
|
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
|
const location = response.headers.get("location");
|
|
if (!location) throw new Error("Album link redirect had no target");
|
|
current = new URL(location, url).toString();
|
|
continue;
|
|
}
|
|
if (response.status === 404) throw new Error("Album not found. Was the share link removed?");
|
|
if (!response.ok) throw new Error(`Google Photos returned HTTP ${response.status}`);
|
|
return new TextDecoder().decode(await readLimited(response, 20 * 1024 * 1024));
|
|
}
|
|
throw new Error("Album link redirected too many times");
|
|
}
|
|
|
|
export function parseAlbumPage(html: string): ScrapedAlbum {
|
|
const match = /AF_initDataCallback\(\{key: 'ds:1'[\s\S]*?data:([\s\S]*?), sideChannel: \{\}\}\);<\/script>/.exec(html);
|
|
if (!match) throw new Error("Could not find the photo list on the album page (Google may have changed the page format)");
|
|
let data: unknown;
|
|
try {
|
|
data = JSON.parse(match[1]);
|
|
} catch {
|
|
throw new Error("Could not read the photo list on the album page (Google may have changed the page format)");
|
|
}
|
|
if (!Array.isArray(data) || !Array.isArray(data[1])) throw new Error("Unexpected album data format");
|
|
const meta = Array.isArray(data[3]) ? data[3] : [];
|
|
const items: ScrapedItem[] = [];
|
|
for (const raw of data[1] as unknown[]) {
|
|
if (!Array.isArray(raw) || typeof raw[0] !== "string" || !Array.isArray(raw[1])) continue;
|
|
const media = raw[1] as unknown[];
|
|
const baseUrl = String(media[0] ?? "");
|
|
let host = "";
|
|
try {
|
|
host = new URL(baseUrl).hostname;
|
|
} catch {
|
|
continue;
|
|
}
|
|
if (host !== "lh3.googleusercontent.com") continue;
|
|
const extras = raw[raw.length - 1];
|
|
items.push({
|
|
googleId: raw[0],
|
|
baseUrl,
|
|
width: Number(media[1]) || 0,
|
|
height: Number(media[2]) || 0,
|
|
takenAt: typeof raw[2] === "number" ? raw[2] : null,
|
|
video: Boolean(extras && typeof extras === "object" && !Array.isArray(extras) && VIDEO_MARKER in extras),
|
|
});
|
|
}
|
|
return {
|
|
title: typeof meta[1] === "string" ? meta[1] : "",
|
|
items,
|
|
hasMorePages: typeof data[2] === "string" && data[2].length > 0,
|
|
};
|
|
}
|
|
|
|
export async function scrapeAlbum(shareUrl: string) {
|
|
return parseAlbumPage(await fetchAlbumPage(shareUrl));
|
|
}
|
|
|
|
async function downloadImage(baseUrl: string, edge: number, maxBytes: number) {
|
|
const url = new URL(`${baseUrl}=w${edge}-h${edge}`);
|
|
if (url.hostname !== "lh3.googleusercontent.com") throw new Error("Unexpected image host");
|
|
const response = await fetch(url, { headers: { "User-Agent": USER_AGENT }, redirect: "error", signal: AbortSignal.timeout(30_000) });
|
|
if (!response.ok) throw new Error(`Image download failed (HTTP ${response.status})`);
|
|
const data = await readLimited(response, maxBytes);
|
|
if (!isJpeg(data)) throw new Error("Image was not a JPEG");
|
|
return data;
|
|
}
|
|
|
|
const photoKey = (googleId: string) => createHash("sha1").update(googleId).digest("hex").slice(0, 20);
|
|
|
|
// ---------- syncing ----------
|
|
|
|
const syncing = new Set<string>();
|
|
export const isSyncing = (albumId: string) => syncing.has(albumId);
|
|
|
|
async function mapLimit<T>(values: T[], limit: number, task: (value: T) => Promise<void>) {
|
|
let next = 0;
|
|
const workers = Array.from({ length: Math.min(limit, values.length) }, async () => {
|
|
while (next < values.length) {
|
|
const value = values[next];
|
|
next += 1;
|
|
await task(value);
|
|
}
|
|
});
|
|
await Promise.all(workers);
|
|
}
|
|
|
|
export async function syncAlbum(album: PhotoAlbum) {
|
|
if (syncing.has(album.id)) return readIndex(album.id);
|
|
syncing.add(album.id);
|
|
const index = await readIndex(album.id);
|
|
const now = Date.now();
|
|
index.lastAttemptAt = now;
|
|
if (!index.firstAttemptAt) index.firstAttemptAt = now;
|
|
try {
|
|
const scraped = await scrapeAlbum(album.shareUrl);
|
|
const wanted = scraped.items.filter((item) => !item.video).slice(0, MAX_PHOTOS_PER_ALBUM);
|
|
if (!wanted.length && index.photos.length) throw new Error("Album page listed no photos; keeping the cached ones");
|
|
await mkdir(dirFor(album.id), { recursive: true, mode: 0o700 });
|
|
const existing = new Set((await readdir(dirFor(album.id))).filter(validPhotoFile));
|
|
const photos: CachedPhoto[] = [];
|
|
const failures: string[] = [];
|
|
await mapLimit(wanted, 4, async (item) => {
|
|
const key = photoKey(item.googleId);
|
|
try {
|
|
if (!existing.has(`${key}.jpg`)) await writeFileAtomic(photoFilePath(album.id, `${key}.jpg`), await downloadImage(item.baseUrl, PHOTO_EDGE, 15 * 1024 * 1024));
|
|
if (!existing.has(`${key}.t.jpg`)) await writeFileAtomic(photoFilePath(album.id, `${key}.t.jpg`), await downloadImage(item.baseUrl, THUMB_EDGE, 512 * 1024));
|
|
photos.push({ key, takenAt: item.takenAt, width: item.width, height: item.height });
|
|
} catch (error) {
|
|
failures.push(error instanceof Error ? error.message : String(error));
|
|
}
|
|
});
|
|
const keep = new Set(photos.flatMap((photo) => [`${photo.key}.jpg`, `${photo.key}.t.jpg`]));
|
|
for (const file of existing) if (!keep.has(file)) await unlink(photoFilePath(album.id, file)).catch(() => undefined);
|
|
|
|
const order = new Map(wanted.map((item, position) => [photoKey(item.googleId), position]));
|
|
photos.sort((left, right) => (order.get(left.key) ?? 0) - (order.get(right.key) ?? 0));
|
|
index.title = scraped.title;
|
|
index.photos = photos;
|
|
index.truncated = scraped.hasMorePages || scraped.items.length > MAX_PHOTOS_PER_ALBUM;
|
|
if (failures.length && !photos.length) throw new Error(`Every download failed: ${failures[0]}`);
|
|
index.lastSuccessAt = Date.now();
|
|
index.lastError = failures.length ? `${failures.length} photo(s) failed to download: ${failures[0]}` : "";
|
|
console.log(`[photos] synced "${scraped.title}" (${album.id.slice(0, 8)}): ${photos.length} photos${failures.length ? `, ${failures.length} failed` : ""}`);
|
|
} catch (error) {
|
|
index.lastError = error instanceof Error ? error.message : String(error);
|
|
console.warn(`[photos] sync failed for ${album.id.slice(0, 8)}: ${index.lastError}`);
|
|
} finally {
|
|
await writeIndex(album.id, index).catch(() => undefined);
|
|
syncing.delete(album.id);
|
|
}
|
|
return index;
|
|
}
|
|
|
|
/** Deletes cache folders of albums that are no longer in settings. */
|
|
export async function removeOrphanedAlbumFolders() {
|
|
const { photos } = await readSettings();
|
|
await mkdir(photosRoot(), { recursive: true, mode: 0o700 });
|
|
const known = new Set(photos.albums.map((album) => album.id));
|
|
for (const entry of await readdir(photosRoot())) {
|
|
if (entry !== UPLOADS_DIR && validPhotoDir(entry) && !known.has(entry) && !syncing.has(entry)) {
|
|
await rm(dirFor(entry), { recursive: true, force: true });
|
|
}
|
|
}
|
|
}
|
|
|
|
let syncAllRunning: Promise<void> | null = null;
|
|
|
|
/** Syncs every enabled album and deletes cache folders of albums that were removed. */
|
|
export function syncAll() {
|
|
if (syncAllRunning) return syncAllRunning;
|
|
syncAllRunning = (async () => {
|
|
await removeOrphanedAlbumFolders();
|
|
const { photos } = await readSettings();
|
|
for (const album of photos.albums.filter((entry) => entry.enabled)) await syncAlbum(album);
|
|
})()
|
|
.catch((error) => console.warn(`[photos] sync run failed: ${error instanceof Error ? error.message : String(error)}`))
|
|
.finally(() => {
|
|
syncAllRunning = null;
|
|
});
|
|
return syncAllRunning;
|
|
}
|
|
|
|
let schedulerStarted = false;
|
|
export function startPhotoScheduler() {
|
|
if (schedulerStarted) return;
|
|
schedulerStarted = true;
|
|
const run = () => void syncAll();
|
|
setTimeout(run, 15_000).unref();
|
|
setInterval(run, SYNC_INTERVAL_MS).unref();
|
|
}
|
|
|
|
// ---------- uploads ----------
|
|
|
|
let uploadQueue: Promise<unknown> = Promise.resolve();
|
|
function serializedUploads<T>(task: () => Promise<T>) {
|
|
const run = uploadQueue.then(task, task);
|
|
uploadQueue = run.catch(() => undefined);
|
|
return run;
|
|
}
|
|
|
|
export function addUpload(photo: Uint8Array, thumb: Uint8Array, meta: { takenAt: number | null; width: number; height: number }) {
|
|
return serializedUploads(async () => {
|
|
const index = await readIndex(UPLOADS_DIR);
|
|
if (index.photos.length >= MAX_UPLOADS) throw new Error(`At most ${MAX_UPLOADS} uploaded photos`);
|
|
const key = createHash("sha1").update(photo).digest("hex").slice(0, 20);
|
|
if (index.photos.some((entry) => entry.key === key)) return { key, duplicate: true };
|
|
await mkdir(dirFor(UPLOADS_DIR), { recursive: true, mode: 0o700 });
|
|
await writeFileAtomic(photoFilePath(UPLOADS_DIR, `${key}.jpg`), photo);
|
|
await writeFileAtomic(photoFilePath(UPLOADS_DIR, `${key}.t.jpg`), thumb);
|
|
index.photos.push({ key, ...meta });
|
|
index.title = "Uploaded photos";
|
|
index.lastSuccessAt = Date.now();
|
|
await writeIndex(UPLOADS_DIR, index);
|
|
return { key, duplicate: false };
|
|
});
|
|
}
|
|
|
|
export function removeUpload(key: string) {
|
|
return serializedUploads(async () => {
|
|
if (!/^[a-f0-9]{20}$/.test(key)) return false;
|
|
const index = await readIndex(UPLOADS_DIR);
|
|
const before = index.photos.length;
|
|
index.photos = index.photos.filter((entry) => entry.key !== key);
|
|
if (index.photos.length === before) return false;
|
|
await writeIndex(UPLOADS_DIR, index);
|
|
for (const file of [`${key}.jpg`, `${key}.t.jpg`]) await unlink(photoFilePath(UPLOADS_DIR, file)).catch(() => undefined);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
// ---------- what the slideshow shows ----------
|
|
|
|
export type SlidePhoto = { src: string; thumb: string; takenAt: number | null; width: number; height: number };
|
|
|
|
export async function currentSlideshow() {
|
|
const settings = await readSettings();
|
|
const { albums, ...display } = settings.photos;
|
|
const sources = albums.filter((album) => album.enabled).map((album) => album.id);
|
|
if (display.includeUploads) sources.push(UPLOADS_DIR);
|
|
const photos: SlidePhoto[] = [];
|
|
const fingerprint = createHash("sha1").update(settings.updatedAt);
|
|
for (const source of sources) {
|
|
const index = await readIndex(source);
|
|
fingerprint.update(`${source}:${index.lastSuccessAt}:${index.photos.length}`);
|
|
for (const photo of index.photos) {
|
|
photos.push({
|
|
src: `/api/photos/img/${source}/${photo.key}.jpg`,
|
|
thumb: `/api/photos/img/${source}/${photo.key}.t.jpg`,
|
|
takenAt: photo.takenAt,
|
|
width: photo.width,
|
|
height: photo.height,
|
|
});
|
|
}
|
|
}
|
|
return { display, photos, version: fingerprint.digest("hex").slice(0, 16) };
|
|
}
|
|
|
|
/** Albums whose last successful sync is older than 48h (or never succeeded within 48h of being added). */
|
|
export async function staleAlbums() {
|
|
const { photos } = await readSettings();
|
|
const now = Date.now();
|
|
const stale: { id: string; title: string; lastSuccessAt: number | null; lastError: string }[] = [];
|
|
for (const album of photos.albums.filter((entry) => entry.enabled)) {
|
|
const index = await readIndex(album.id);
|
|
const reference = index.lastSuccessAt ?? (index.firstAttemptAt || now);
|
|
if (now - reference > STALE_AFTER_MS) stale.push({ id: album.id, title: album.name || index.title, lastSuccessAt: index.lastSuccessAt, lastError: index.lastError });
|
|
}
|
|
return stale;
|
|
}
|