#!/usr/bin/env python3 """SHA256が同じPDFの移動・改名をpapers.dbへ反映する。 PDF内容が変わってSHA256も変わった場合は更新しない。 その場合はdelete_paper.pyでDBレコードを削除し、 通常の登録フローで再登録する。 既定はdry-run。実際に更新する場合は--applyを指定する。 """ from __future__ import annotations import argparse import hashlib import shutil import sqlite3 import sys from collections import defaultdict from dataclasses import dataclass from pathlib import Path from typing import Iterable from common import CONFIG, resolve_pdf_path @dataclass(frozen=True) class PdfFile: relative_path: str sha256: str file_size: int file_mtime: float @dataclass(frozen=True) class PathUpdate: paper_id: int title: str old_path: str new_path: str sha256: str file_size: int file_mtime: float def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="SHA256が同じPDFの移動・改名をDBへ反映する" ) parser.add_argument( "--database", type=Path, default=CONFIG.path.database, ) parser.add_argument( "--pdf-root", type=Path, default=CONFIG.path.pdf_root, ) parser.add_argument( "--scan-all", action="store_true", help="PDF_ROOT全体を走査する", ) parser.add_argument( "--apply", action="store_true", help="更新を実際にDBへ反映する", ) parser.add_argument( "--no-backup", action="store_true", help="更新前バックアップを作成しない", ) return parser.parse_args() def validate_paths(database: Path, pdf_root: Path) -> None: if not database.is_file(): raise FileNotFoundError(database) if not pdf_root.is_dir(): raise NotADirectoryError(pdf_root) def scan_roots( pdf_root: Path, scan_all: bool, ) -> tuple[Path, ...]: if scan_all: return (pdf_root,) roots = tuple( pdf_root / name for name in CONFIG.processing.scan_directories ) for root in roots: if not root.is_dir(): raise NotADirectoryError(root) return roots def iter_pdf_paths( roots: Iterable[Path], ) -> Iterable[Path]: seen: set[Path] = set() for root in roots: for path in root.rglob("*"): if not path.is_file(): continue if path.suffix.lower() != ".pdf": continue resolved = path.resolve() if resolved in seen: continue seen.add(resolved) yield resolved def calculate_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as file: for chunk in iter( lambda: file.read(1024 * 1024), b"", ): digest.update(chunk) return digest.hexdigest() def inspect_pdf( path: Path, pdf_root: Path, ) -> PdfFile: stat = path.stat() return PdfFile( relative_path=path.relative_to( pdf_root ).as_posix(), sha256=calculate_sha256(path), file_size=stat.st_size, file_mtime=stat.st_mtime, ) def build_sha_index( pdf_root: Path, roots: tuple[Path, ...], ) -> dict[str, list[PdfFile]]: index: dict[str, list[PdfFile]] = defaultdict(list) paths = sorted( iter_pdf_paths(roots), key=lambda item: str(item).casefold(), ) for number, path in enumerate(paths, start=1): pdf = inspect_pdf(path, pdf_root) index[pdf.sha256].append(pdf) print( f"\rPDFを走査中: {number}件", end="", flush=True, ) if paths: print() return dict(index) def open_database( database: Path, ) -> sqlite3.Connection: connection = sqlite3.connect(database) connection.row_factory = sqlite3.Row connection.execute("PRAGMA foreign_keys=ON") return connection def load_papers( connection: sqlite3.Connection, ) -> list[sqlite3.Row]: sql = """ SELECT id, title, pdf_path, file_sha256 FROM papers WHERE file_sha256 IS NOT NULL AND TRIM(file_sha256) <> '' ORDER BY id """ return connection.execute(sql).fetchall() def current_path_is_valid( pdf_root: Path, row: sqlite3.Row, ) -> bool: if not row["pdf_path"]: return False try: path = resolve_pdf_path( pdf_root, row["pdf_path"], ) except Exception: return False if not path.is_file(): return False return ( calculate_sha256(path) == row["file_sha256"].strip().lower() ) def find_updates( rows: Iterable[sqlite3.Row], index: dict[str, list[PdfFile]], pdf_root: Path, ) -> tuple[ list[PathUpdate], list[sqlite3.Row], list[sqlite3.Row], ]: updates: list[PathUpdate] = [] ambiguous: list[sqlite3.Row] = [] not_found: list[sqlite3.Row] = [] for row in rows: if current_path_is_valid(pdf_root, row): continue sha256 = row["file_sha256"].strip().lower() candidates = index.get(sha256, []) if not candidates: not_found.append(row) continue if len(candidates) != 1: ambiguous.append(row) continue candidate = candidates[0] updates.append( PathUpdate( paper_id=row["id"], title=row["title"], old_path=row["pdf_path"] or "", new_path=candidate.relative_path, sha256=sha256, file_size=candidate.file_size, file_mtime=candidate.file_mtime, ) ) return updates, ambiguous, not_found def print_updates( updates: Iterable[PathUpdate], ) -> None: items = list(updates) if not items: return print() print("=== 更新候補 ===") for item in items: print(f"id: {item.paper_id}") print(f"title: {item.title}") print(f"old: {item.old_path or '(empty)'}") print(f"new: {item.new_path}") print() def print_rows( title: str, rows: Iterable[sqlite3.Row], ) -> None: items = list(rows) if not items: return print() print(f"=== {title} ===") for row in items: print( f"id={row['id']}: " f"{row['pdf_path']} " f"({row['title']})" ) def create_backup(database: Path) -> Path: backup = database.with_name( database.name + ".before-path-update" ) counter = 1 while backup.exists(): backup = database.with_name( database.name + f".before-path-update.{counter}" ) counter += 1 shutil.copy2(database, backup) return backup def apply_updates( connection: sqlite3.Connection, updates: Iterable[PathUpdate], ) -> int: sql = """ UPDATE papers SET pdf_path = ?, file_size = ?, file_mtime = ? WHERE id = ? AND lower(file_sha256) = ? """ count = 0 with connection: for item in updates: cursor = connection.execute( sql, ( item.new_path, item.file_size, item.file_mtime, item.paper_id, item.sha256, ), ) count += cursor.rowcount return count def main() -> int: args = parse_args() try: database = args.database.expanduser().resolve() pdf_root = args.pdf_root.expanduser().resolve() validate_paths(database, pdf_root) index = build_sha_index( pdf_root, scan_roots(pdf_root, args.scan_all), ) with open_database(database) as connection: rows = load_papers(connection) updates, ambiguous, not_found = ( find_updates( rows, index, pdf_root, ) ) print_updates(updates) print_rows( "同一SHA256の候補が複数", ambiguous, ) print_rows( "SHA256一致ファイルなし", not_found, ) if args.apply and updates: if not args.no_backup: backup = create_backup(database) print(f"DBバックアップ: {backup}") changed = apply_updates( connection, updates, ) print(f"DB更新: {changed} 件") print() print("=== 概要 ===") print( "走査PDF: " f"{sum(len(v) for v in index.values())} 件" ) print(f"DB照合対象: {len(rows)} 件") print(f"更新候補: {len(updates)} 件") print(f"曖昧な一致: {len(ambiguous)} 件") print(f"一致なし: {len(not_found)} 件") print( "実行モード: " + ("更新実行" if args.apply else "dry-run") ) if updates and not args.apply: print() print( "更新するには --apply を指定してください。" ) return 0 except KeyboardInterrupt: return 130 except Exception as exc: print( f"update_pdf_paths.py: {exc}", file=sys.stderr, ) return 1 if __name__ == "__main__": raise SystemExit(main())