import { createHash } from "node:crypto"; import { mkdir, readdir, readFile, rename, rm, unlink, writeFile } from "node:fs/promises"; import path from "node:path"; import { type PhotoAlbum, readSettings } from "./store"; /* * Google Photos shared albums. Google removed third-party read access to user albums from the * Library API on 2025-03-31, so this reads the public share page the same way a browser does: * the album's items are embedded as JSON in an AF_initDataCallback({key: 'ds:1', ...}) block. * This is unofficial and may break when Google changes the page. Cached photos keep showing * if it does, and the album reports lastError plus a stale lastSuccessAt. */ export const MAX_PHOTOS_PER_ALBUM = 500; export const MAX_UPLOADS = 500; export const SYNC_INTERVAL_MS = 6 * 60 * 60 * 1000; export const STALE_AFTER_MS = 48 * 60 * 60 * 1000; const PHOTO_EDGE = 1920; const THUMB_EDGE = 64; const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"; const VIDEO_MARKER = "76647426"; export type CachedPhoto = { key: string; takenAt: number | null; width: number; height: number }; export type AlbumIndex = { title: string; firstAttemptAt: number; lastAttemptAt: number; lastSuccessAt: number | null; lastError: string; truncated: boolean; photos: CachedPhoto[]; }; const photosRoot = () => path.join(process.env.ORGANIZER_DATA_DIR || "/data", "photos"); export const UPLOADS_DIR = "uploads"; const dirFor = (name: string) => path.join(photosRoot(), name); const indexPath = (name: string) => path.join(dirFor(name), "index.json"); export function validPhotoDir(name: string) { return name === UPLOADS_DIR || /^[A-Za-z0-9-]{8,64}$/.test(name); } export function validPhotoFile(name: string) { return /^[a-f0-9]{20}(\.t)?\.jpg$/.test(name); } export const photoFilePath = (dir: string, file: string) => path.join(dirFor(dir), file); function emptyIndex(): AlbumIndex { return { title: "", firstAttemptAt: 0, lastAttemptAt: 0, lastSuccessAt: null, lastError: "", truncated: false, photos: [] }; } export async function readIndex(name: string): Promise { try { return { ...emptyIndex(), ...(JSON.parse(await readFile(indexPath(name), "utf8")) as Partial) }; } catch { return emptyIndex(); } } async function writeIndex(name: string, index: AlbumIndex) { await mkdir(dirFor(name), { recursive: true, mode: 0o700 }); const temporary = `${indexPath(name)}.${process.pid}.tmp`; await writeFile(temporary, JSON.stringify(index), { mode: 0o600 }); await rename(temporary, indexPath(name)); } async function writeFileAtomic(file: string, data: Uint8Array) { const temporary = `${file}.${process.pid}.tmp`; await writeFile(temporary, data, { mode: 0o600 }); await rename(temporary, file); } export function isJpeg(data: Uint8Array) { return data.length > 4 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff; } async function readLimited(response: Response, maxBytes: number) { const declared = Number(response.headers.get("content-length") ?? 0); if (declared > maxBytes) throw new Error("Response too large"); if (!response.body) return new Uint8Array(); const reader = response.body.getReader(); const chunks: Uint8Array[] = []; let total = 0; while (true) { const { done, value } = await reader.read(); if (done) break; total += value.byteLength; if (total > maxBytes) { await reader.cancel(); throw new Error("Response too large"); } chunks.push(value); } const out = new Uint8Array(total); let offset = 0; for (const chunk of chunks) { out.set(chunk, offset); offset += chunk.byteLength; } return out; } // ---------- scraping ---------- type ScrapedItem = { googleId: string; baseUrl: string; width: number; height: number; takenAt: number | null; video: boolean }; export type ScrapedAlbum = { title: string; items: ScrapedItem[]; hasMorePages: boolean }; const ALLOWED_PAGE_HOSTS = new Set(["photos.app.goo.gl", "photos.google.com"]); async function fetchAlbumPage(shareUrl: string) { let current = shareUrl; for (let hop = 0; hop <= 5; hop += 1) { const url = new URL(current); if (url.protocol !== "https:" || !ALLOWED_PAGE_HOSTS.has(url.hostname)) throw new Error(`Album link redirected to an unexpected host (${url.hostname})`); const response = await fetch(url, { headers: { "User-Agent": USER_AGENT, "Accept-Language": "en-US,en;q=0.9", Accept: "text/html" }, redirect: "manual", signal: AbortSignal.timeout(20_000), }); if ([301, 302, 303, 307, 308].includes(response.status)) { const location = response.headers.get("location"); if (!location) throw new Error("Album link redirect had no target"); current = new URL(location, url).toString(); continue; } if (response.status === 404) throw new Error("Album not found. Was the share link removed?"); if (!response.ok) throw new Error(`Google Photos returned HTTP ${response.status}`); return new TextDecoder().decode(await readLimited(response, 20 * 1024 * 1024)); } throw new Error("Album link redirected too many times"); } export function parseAlbumPage(html: string): ScrapedAlbum { const match = /AF_initDataCallback\(\{key: 'ds:1'[\s\S]*?data:([\s\S]*?), sideChannel: \{\}\}\);<\/script>/.exec(html); if (!match) throw new Error("Could not find the photo list on the album page (Google may have changed the page format)"); let data: unknown; try { data = JSON.parse(match[1]); } catch { throw new Error("Could not read the photo list on the album page (Google may have changed the page format)"); } if (!Array.isArray(data) || !Array.isArray(data[1])) throw new Error("Unexpected album data format"); const meta = Array.isArray(data[3]) ? data[3] : []; const items: ScrapedItem[] = []; for (const raw of data[1] as unknown[]) { if (!Array.isArray(raw) || typeof raw[0] !== "string" || !Array.isArray(raw[1])) continue; const media = raw[1] as unknown[]; const baseUrl = String(media[0] ?? ""); let host = ""; try { host = new URL(baseUrl).hostname; } catch { continue; } if (host !== "lh3.googleusercontent.com") continue; const extras = raw[raw.length - 1]; items.push({ googleId: raw[0], baseUrl, width: Number(media[1]) || 0, height: Number(media[2]) || 0, takenAt: typeof raw[2] === "number" ? raw[2] : null, video: Boolean(extras && typeof extras === "object" && !Array.isArray(extras) && VIDEO_MARKER in extras), }); } return { title: typeof meta[1] === "string" ? meta[1] : "", items, hasMorePages: typeof data[2] === "string" && data[2].length > 0, }; } export async function scrapeAlbum(shareUrl: string) { return parseAlbumPage(await fetchAlbumPage(shareUrl)); } async function downloadImage(baseUrl: string, edge: number, maxBytes: number) { const url = new URL(`${baseUrl}=w${edge}-h${edge}`); if (url.hostname !== "lh3.googleusercontent.com") throw new Error("Unexpected image host"); const response = await fetch(url, { headers: { "User-Agent": USER_AGENT }, redirect: "error", signal: AbortSignal.timeout(30_000) }); if (!response.ok) throw new Error(`Image download failed (HTTP ${response.status})`); const data = await readLimited(response, maxBytes); if (!isJpeg(data)) throw new Error("Image was not a JPEG"); return data; } const photoKey = (googleId: string) => createHash("sha1").update(googleId).digest("hex").slice(0, 20); // ---------- syncing ---------- const syncing = new Set(); export const isSyncing = (albumId: string) => syncing.has(albumId); async function mapLimit(values: T[], limit: number, task: (value: T) => Promise) { let next = 0; const workers = Array.from({ length: Math.min(limit, values.length) }, async () => { while (next < values.length) { const value = values[next]; next += 1; await task(value); } }); await Promise.all(workers); } export async function syncAlbum(album: PhotoAlbum) { if (syncing.has(album.id)) return readIndex(album.id); syncing.add(album.id); const index = await readIndex(album.id); const now = Date.now(); index.lastAttemptAt = now; if (!index.firstAttemptAt) index.firstAttemptAt = now; try { const scraped = await scrapeAlbum(album.shareUrl); const wanted = scraped.items.filter((item) => !item.video).slice(0, MAX_PHOTOS_PER_ALBUM); if (!wanted.length && index.photos.length) throw new Error("Album page listed no photos; keeping the cached ones"); await mkdir(dirFor(album.id), { recursive: true, mode: 0o700 }); const existing = new Set((await readdir(dirFor(album.id))).filter(validPhotoFile)); const photos: CachedPhoto[] = []; const failures: string[] = []; await mapLimit(wanted, 4, async (item) => { const key = photoKey(item.googleId); try { if (!existing.has(`${key}.jpg`)) await writeFileAtomic(photoFilePath(album.id, `${key}.jpg`), await downloadImage(item.baseUrl, PHOTO_EDGE, 15 * 1024 * 1024)); if (!existing.has(`${key}.t.jpg`)) await writeFileAtomic(photoFilePath(album.id, `${key}.t.jpg`), await downloadImage(item.baseUrl, THUMB_EDGE, 512 * 1024)); photos.push({ key, takenAt: item.takenAt, width: item.width, height: item.height }); } catch (error) { failures.push(error instanceof Error ? error.message : String(error)); } }); const keep = new Set(photos.flatMap((photo) => [`${photo.key}.jpg`, `${photo.key}.t.jpg`])); for (const file of existing) if (!keep.has(file)) await unlink(photoFilePath(album.id, file)).catch(() => undefined); const order = new Map(wanted.map((item, position) => [photoKey(item.googleId), position])); photos.sort((left, right) => (order.get(left.key) ?? 0) - (order.get(right.key) ?? 0)); index.title = scraped.title; index.photos = photos; index.truncated = scraped.hasMorePages || scraped.items.length > MAX_PHOTOS_PER_ALBUM; if (failures.length && !photos.length) throw new Error(`Every download failed: ${failures[0]}`); index.lastSuccessAt = Date.now(); index.lastError = failures.length ? `${failures.length} photo(s) failed to download: ${failures[0]}` : ""; console.log(`[photos] synced "${scraped.title}" (${album.id.slice(0, 8)}): ${photos.length} photos${failures.length ? `, ${failures.length} failed` : ""}`); } catch (error) { index.lastError = error instanceof Error ? error.message : String(error); console.warn(`[photos] sync failed for ${album.id.slice(0, 8)}: ${index.lastError}`); } finally { await writeIndex(album.id, index).catch(() => undefined); syncing.delete(album.id); } return index; } /** Deletes cache folders of albums that are no longer in settings. */ export async function removeOrphanedAlbumFolders() { const { photos } = await readSettings(); await mkdir(photosRoot(), { recursive: true, mode: 0o700 }); const known = new Set(photos.albums.map((album) => album.id)); for (const entry of await readdir(photosRoot())) { if (entry !== UPLOADS_DIR && validPhotoDir(entry) && !known.has(entry) && !syncing.has(entry)) { await rm(dirFor(entry), { recursive: true, force: true }); } } } let syncAllRunning: Promise | null = null; /** Syncs every enabled album and deletes cache folders of albums that were removed. */ export function syncAll() { if (syncAllRunning) return syncAllRunning; syncAllRunning = (async () => { await removeOrphanedAlbumFolders(); const { photos } = await readSettings(); for (const album of photos.albums.filter((entry) => entry.enabled)) await syncAlbum(album); })() .catch((error) => console.warn(`[photos] sync run failed: ${error instanceof Error ? error.message : String(error)}`)) .finally(() => { syncAllRunning = null; }); return syncAllRunning; } let schedulerStarted = false; export function startPhotoScheduler() { if (schedulerStarted) return; schedulerStarted = true; const run = () => void syncAll(); setTimeout(run, 15_000).unref(); setInterval(run, SYNC_INTERVAL_MS).unref(); } // ---------- uploads ---------- let uploadQueue: Promise = Promise.resolve(); function serializedUploads(task: () => Promise) { const run = uploadQueue.then(task, task); uploadQueue = run.catch(() => undefined); return run; } export function addUpload(photo: Uint8Array, thumb: Uint8Array, meta: { takenAt: number | null; width: number; height: number }) { return serializedUploads(async () => { const index = await readIndex(UPLOADS_DIR); if (index.photos.length >= MAX_UPLOADS) throw new Error(`At most ${MAX_UPLOADS} uploaded photos`); const key = createHash("sha1").update(photo).digest("hex").slice(0, 20); if (index.photos.some((entry) => entry.key === key)) return { key, duplicate: true }; await mkdir(dirFor(UPLOADS_DIR), { recursive: true, mode: 0o700 }); await writeFileAtomic(photoFilePath(UPLOADS_DIR, `${key}.jpg`), photo); await writeFileAtomic(photoFilePath(UPLOADS_DIR, `${key}.t.jpg`), thumb); index.photos.push({ key, ...meta }); index.title = "Uploaded photos"; index.lastSuccessAt = Date.now(); await writeIndex(UPLOADS_DIR, index); return { key, duplicate: false }; }); } export function removeUpload(key: string) { return serializedUploads(async () => { if (!/^[a-f0-9]{20}$/.test(key)) return false; const index = await readIndex(UPLOADS_DIR); const before = index.photos.length; index.photos = index.photos.filter((entry) => entry.key !== key); if (index.photos.length === before) return false; await writeIndex(UPLOADS_DIR, index); for (const file of [`${key}.jpg`, `${key}.t.jpg`]) await unlink(photoFilePath(UPLOADS_DIR, file)).catch(() => undefined); return true; }); } // ---------- what the slideshow shows ---------- export type SlidePhoto = { src: string; thumb: string; takenAt: number | null; width: number; height: number }; export async function currentSlideshow() { const settings = await readSettings(); const { albums, ...display } = settings.photos; const sources = albums.filter((album) => album.enabled).map((album) => album.id); if (display.includeUploads) sources.push(UPLOADS_DIR); const photos: SlidePhoto[] = []; const fingerprint = createHash("sha1").update(settings.updatedAt); for (const source of sources) { const index = await readIndex(source); fingerprint.update(`${source}:${index.lastSuccessAt}:${index.photos.length}`); for (const photo of index.photos) { photos.push({ src: `/api/photos/img/${source}/${photo.key}.jpg`, thumb: `/api/photos/img/${source}/${photo.key}.t.jpg`, takenAt: photo.takenAt, width: photo.width, height: photo.height, }); } } return { display, photos, version: fingerprint.digest("hex").slice(0, 16) }; } /** Albums whose last successful sync is older than 48h (or never succeeded within 48h of being added). */ export async function staleAlbums() { const { photos } = await readSettings(); const now = Date.now(); const stale: { id: string; title: string; lastSuccessAt: number | null; lastError: string }[] = []; for (const album of photos.albums.filter((entry) => entry.enabled)) { const index = await readIndex(album.id); const reference = index.lastSuccessAt ?? (index.firstAttemptAt || now); if (now - reference > STALE_AFTER_MS) stale.push({ id: album.id, title: album.name || index.title, lastSuccessAt: index.lastSuccessAt, lastError: index.lastError }); } return stale; }