443 lines
9.7 KiB
Python
Executable File
443 lines
9.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""SHA256が同じPDFの移動・改名をpapers.dbへ反映する。
|
|
|
|
PDF内容が変わってSHA256も変わった場合は更新しない。
|
|
その場合はdelete_paper.pyでDBレコードを削除し、
|
|
通常の登録フローで再登録する。
|
|
|
|
既定はdry-run。実際に更新する場合は--applyを指定する。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
from common import CONFIG, resolve_pdf_path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PdfFile:
|
|
relative_path: str
|
|
sha256: str
|
|
file_size: int
|
|
file_mtime: float
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PathUpdate:
|
|
paper_id: int
|
|
title: str
|
|
old_path: str
|
|
new_path: str
|
|
sha256: str
|
|
file_size: int
|
|
file_mtime: float
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="SHA256が同じPDFの移動・改名をDBへ反映する"
|
|
)
|
|
parser.add_argument(
|
|
"--database",
|
|
type=Path,
|
|
default=CONFIG.path.database,
|
|
)
|
|
parser.add_argument(
|
|
"--pdf-root",
|
|
type=Path,
|
|
default=CONFIG.path.pdf_root,
|
|
)
|
|
parser.add_argument(
|
|
"--scan-all",
|
|
action="store_true",
|
|
help="PDF_ROOT全体を走査する",
|
|
)
|
|
parser.add_argument(
|
|
"--apply",
|
|
action="store_true",
|
|
help="更新を実際にDBへ反映する",
|
|
)
|
|
parser.add_argument(
|
|
"--no-backup",
|
|
action="store_true",
|
|
help="更新前バックアップを作成しない",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def validate_paths(database: Path, pdf_root: Path) -> None:
|
|
if not database.is_file():
|
|
raise FileNotFoundError(database)
|
|
|
|
if not pdf_root.is_dir():
|
|
raise NotADirectoryError(pdf_root)
|
|
|
|
|
|
def scan_roots(
|
|
pdf_root: Path,
|
|
scan_all: bool,
|
|
) -> tuple[Path, ...]:
|
|
if scan_all:
|
|
return (pdf_root,)
|
|
|
|
roots = tuple(
|
|
pdf_root / name
|
|
for name in CONFIG.processing.scan_directories
|
|
)
|
|
|
|
for root in roots:
|
|
if not root.is_dir():
|
|
raise NotADirectoryError(root)
|
|
|
|
return roots
|
|
|
|
|
|
def iter_pdf_paths(
|
|
roots: Iterable[Path],
|
|
) -> Iterable[Path]:
|
|
seen: set[Path] = set()
|
|
|
|
for root in roots:
|
|
for path in root.rglob("*"):
|
|
if not path.is_file():
|
|
continue
|
|
|
|
if path.suffix.lower() != ".pdf":
|
|
continue
|
|
|
|
resolved = path.resolve()
|
|
|
|
if resolved in seen:
|
|
continue
|
|
|
|
seen.add(resolved)
|
|
yield resolved
|
|
|
|
|
|
def calculate_sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
|
|
with path.open("rb") as file:
|
|
for chunk in iter(
|
|
lambda: file.read(1024 * 1024),
|
|
b"",
|
|
):
|
|
digest.update(chunk)
|
|
|
|
return digest.hexdigest()
|
|
|
|
|
|
def inspect_pdf(
|
|
path: Path,
|
|
pdf_root: Path,
|
|
) -> PdfFile:
|
|
stat = path.stat()
|
|
|
|
return PdfFile(
|
|
relative_path=path.relative_to(
|
|
pdf_root
|
|
).as_posix(),
|
|
sha256=calculate_sha256(path),
|
|
file_size=stat.st_size,
|
|
file_mtime=stat.st_mtime,
|
|
)
|
|
|
|
|
|
def build_sha_index(
|
|
pdf_root: Path,
|
|
roots: tuple[Path, ...],
|
|
) -> dict[str, list[PdfFile]]:
|
|
index: dict[str, list[PdfFile]] = defaultdict(list)
|
|
|
|
paths = sorted(
|
|
iter_pdf_paths(roots),
|
|
key=lambda item: str(item).casefold(),
|
|
)
|
|
|
|
for number, path in enumerate(paths, start=1):
|
|
pdf = inspect_pdf(path, pdf_root)
|
|
index[pdf.sha256].append(pdf)
|
|
|
|
print(
|
|
f"\rPDFを走査中: {number}件",
|
|
end="",
|
|
flush=True,
|
|
)
|
|
|
|
if paths:
|
|
print()
|
|
|
|
return dict(index)
|
|
|
|
|
|
def open_database(
|
|
database: Path,
|
|
) -> sqlite3.Connection:
|
|
connection = sqlite3.connect(database)
|
|
connection.row_factory = sqlite3.Row
|
|
connection.execute("PRAGMA foreign_keys=ON")
|
|
return connection
|
|
|
|
|
|
def load_papers(
|
|
connection: sqlite3.Connection,
|
|
) -> list[sqlite3.Row]:
|
|
sql = """
|
|
SELECT id, title, pdf_path, file_sha256
|
|
FROM papers
|
|
WHERE file_sha256 IS NOT NULL
|
|
AND TRIM(file_sha256) <> ''
|
|
ORDER BY id
|
|
"""
|
|
return connection.execute(sql).fetchall()
|
|
|
|
|
|
def current_path_is_valid(
|
|
pdf_root: Path,
|
|
row: sqlite3.Row,
|
|
) -> bool:
|
|
if not row["pdf_path"]:
|
|
return False
|
|
|
|
try:
|
|
path = resolve_pdf_path(
|
|
pdf_root,
|
|
row["pdf_path"],
|
|
)
|
|
except Exception:
|
|
return False
|
|
|
|
if not path.is_file():
|
|
return False
|
|
|
|
return (
|
|
calculate_sha256(path)
|
|
== row["file_sha256"].strip().lower()
|
|
)
|
|
|
|
|
|
def find_updates(
|
|
rows: Iterable[sqlite3.Row],
|
|
index: dict[str, list[PdfFile]],
|
|
pdf_root: Path,
|
|
) -> tuple[
|
|
list[PathUpdate],
|
|
list[sqlite3.Row],
|
|
list[sqlite3.Row],
|
|
]:
|
|
updates: list[PathUpdate] = []
|
|
ambiguous: list[sqlite3.Row] = []
|
|
not_found: list[sqlite3.Row] = []
|
|
|
|
for row in rows:
|
|
if current_path_is_valid(pdf_root, row):
|
|
continue
|
|
|
|
sha256 = row["file_sha256"].strip().lower()
|
|
candidates = index.get(sha256, [])
|
|
|
|
if not candidates:
|
|
not_found.append(row)
|
|
continue
|
|
|
|
if len(candidates) != 1:
|
|
ambiguous.append(row)
|
|
continue
|
|
|
|
candidate = candidates[0]
|
|
|
|
updates.append(
|
|
PathUpdate(
|
|
paper_id=row["id"],
|
|
title=row["title"],
|
|
old_path=row["pdf_path"] or "",
|
|
new_path=candidate.relative_path,
|
|
sha256=sha256,
|
|
file_size=candidate.file_size,
|
|
file_mtime=candidate.file_mtime,
|
|
)
|
|
)
|
|
|
|
return updates, ambiguous, not_found
|
|
|
|
|
|
def print_updates(
|
|
updates: Iterable[PathUpdate],
|
|
) -> None:
|
|
items = list(updates)
|
|
|
|
if not items:
|
|
return
|
|
|
|
print()
|
|
print("=== 更新候補 ===")
|
|
|
|
for item in items:
|
|
print(f"id: {item.paper_id}")
|
|
print(f"title: {item.title}")
|
|
print(f"old: {item.old_path or '(empty)'}")
|
|
print(f"new: {item.new_path}")
|
|
print()
|
|
|
|
|
|
def print_rows(
|
|
title: str,
|
|
rows: Iterable[sqlite3.Row],
|
|
) -> None:
|
|
items = list(rows)
|
|
|
|
if not items:
|
|
return
|
|
|
|
print()
|
|
print(f"=== {title} ===")
|
|
|
|
for row in items:
|
|
print(
|
|
f"id={row['id']}: "
|
|
f"{row['pdf_path']} "
|
|
f"({row['title']})"
|
|
)
|
|
|
|
|
|
def create_backup(database: Path) -> Path:
|
|
backup = database.with_name(
|
|
database.name + ".before-path-update"
|
|
)
|
|
|
|
counter = 1
|
|
|
|
while backup.exists():
|
|
backup = database.with_name(
|
|
database.name
|
|
+ f".before-path-update.{counter}"
|
|
)
|
|
counter += 1
|
|
|
|
shutil.copy2(database, backup)
|
|
return backup
|
|
|
|
|
|
def apply_updates(
|
|
connection: sqlite3.Connection,
|
|
updates: Iterable[PathUpdate],
|
|
) -> int:
|
|
sql = """
|
|
UPDATE papers
|
|
SET pdf_path = ?,
|
|
file_size = ?,
|
|
file_mtime = ?
|
|
WHERE id = ?
|
|
AND lower(file_sha256) = ?
|
|
"""
|
|
|
|
count = 0
|
|
|
|
with connection:
|
|
for item in updates:
|
|
cursor = connection.execute(
|
|
sql,
|
|
(
|
|
item.new_path,
|
|
item.file_size,
|
|
item.file_mtime,
|
|
item.paper_id,
|
|
item.sha256,
|
|
),
|
|
)
|
|
count += cursor.rowcount
|
|
|
|
return count
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
|
|
try:
|
|
database = args.database.expanduser().resolve()
|
|
pdf_root = args.pdf_root.expanduser().resolve()
|
|
|
|
validate_paths(database, pdf_root)
|
|
|
|
index = build_sha_index(
|
|
pdf_root,
|
|
scan_roots(pdf_root, args.scan_all),
|
|
)
|
|
|
|
with open_database(database) as connection:
|
|
rows = load_papers(connection)
|
|
updates, ambiguous, not_found = (
|
|
find_updates(
|
|
rows,
|
|
index,
|
|
pdf_root,
|
|
)
|
|
)
|
|
|
|
print_updates(updates)
|
|
print_rows(
|
|
"同一SHA256の候補が複数",
|
|
ambiguous,
|
|
)
|
|
print_rows(
|
|
"SHA256一致ファイルなし",
|
|
not_found,
|
|
)
|
|
|
|
if args.apply and updates:
|
|
if not args.no_backup:
|
|
backup = create_backup(database)
|
|
print(f"DBバックアップ: {backup}")
|
|
|
|
changed = apply_updates(
|
|
connection,
|
|
updates,
|
|
)
|
|
print(f"DB更新: {changed} 件")
|
|
|
|
print()
|
|
print("=== 概要 ===")
|
|
print(
|
|
"走査PDF: "
|
|
f"{sum(len(v) for v in index.values())} 件"
|
|
)
|
|
print(f"DB照合対象: {len(rows)} 件")
|
|
print(f"更新候補: {len(updates)} 件")
|
|
print(f"曖昧な一致: {len(ambiguous)} 件")
|
|
print(f"一致なし: {len(not_found)} 件")
|
|
print(
|
|
"実行モード: "
|
|
+ ("更新実行" if args.apply else "dry-run")
|
|
)
|
|
|
|
if updates and not args.apply:
|
|
print()
|
|
print(
|
|
"更新するには --apply を指定してください。"
|
|
)
|
|
|
|
return 0
|
|
|
|
except KeyboardInterrupt:
|
|
return 130
|
|
|
|
except Exception as exc:
|
|
print(
|
|
f"update_pdf_paths.py: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|