Initial commit
This commit is contained in:
Executable
+442
@@ -0,0 +1,442 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SHA256が同じPDFの移動・改名をpapers.dbへ反映する。
|
||||
|
||||
PDF内容が変わってSHA256も変わった場合は更新しない。
|
||||
その場合はdelete_paper.pyでDBレコードを削除し、
|
||||
通常の登録フローで再登録する。
|
||||
|
||||
既定はdry-run。実際に更新する場合は--applyを指定する。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from common import CONFIG, resolve_pdf_path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfFile:
|
||||
relative_path: str
|
||||
sha256: str
|
||||
file_size: int
|
||||
file_mtime: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PathUpdate:
|
||||
paper_id: int
|
||||
title: str
|
||||
old_path: str
|
||||
new_path: str
|
||||
sha256: str
|
||||
file_size: int
|
||||
file_mtime: float
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="SHA256が同じPDFの移動・改名をDBへ反映する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database",
|
||||
type=Path,
|
||||
default=CONFIG.path.database,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pdf-root",
|
||||
type=Path,
|
||||
default=CONFIG.path.pdf_root,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scan-all",
|
||||
action="store_true",
|
||||
help="PDF_ROOT全体を走査する",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="更新を実際にDBへ反映する",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-backup",
|
||||
action="store_true",
|
||||
help="更新前バックアップを作成しない",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_paths(database: Path, pdf_root: Path) -> None:
|
||||
if not database.is_file():
|
||||
raise FileNotFoundError(database)
|
||||
|
||||
if not pdf_root.is_dir():
|
||||
raise NotADirectoryError(pdf_root)
|
||||
|
||||
|
||||
def scan_roots(
|
||||
pdf_root: Path,
|
||||
scan_all: bool,
|
||||
) -> tuple[Path, ...]:
|
||||
if scan_all:
|
||||
return (pdf_root,)
|
||||
|
||||
roots = tuple(
|
||||
pdf_root / name
|
||||
for name in CONFIG.processing.scan_directories
|
||||
)
|
||||
|
||||
for root in roots:
|
||||
if not root.is_dir():
|
||||
raise NotADirectoryError(root)
|
||||
|
||||
return roots
|
||||
|
||||
|
||||
def iter_pdf_paths(
|
||||
roots: Iterable[Path],
|
||||
) -> Iterable[Path]:
|
||||
seen: set[Path] = set()
|
||||
|
||||
for root in roots:
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
|
||||
if path.suffix.lower() != ".pdf":
|
||||
continue
|
||||
|
||||
resolved = path.resolve()
|
||||
|
||||
if resolved in seen:
|
||||
continue
|
||||
|
||||
seen.add(resolved)
|
||||
yield resolved
|
||||
|
||||
|
||||
def calculate_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
|
||||
with path.open("rb") as file:
|
||||
for chunk in iter(
|
||||
lambda: file.read(1024 * 1024),
|
||||
b"",
|
||||
):
|
||||
digest.update(chunk)
|
||||
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def inspect_pdf(
|
||||
path: Path,
|
||||
pdf_root: Path,
|
||||
) -> PdfFile:
|
||||
stat = path.stat()
|
||||
|
||||
return PdfFile(
|
||||
relative_path=path.relative_to(
|
||||
pdf_root
|
||||
).as_posix(),
|
||||
sha256=calculate_sha256(path),
|
||||
file_size=stat.st_size,
|
||||
file_mtime=stat.st_mtime,
|
||||
)
|
||||
|
||||
|
||||
def build_sha_index(
|
||||
pdf_root: Path,
|
||||
roots: tuple[Path, ...],
|
||||
) -> dict[str, list[PdfFile]]:
|
||||
index: dict[str, list[PdfFile]] = defaultdict(list)
|
||||
|
||||
paths = sorted(
|
||||
iter_pdf_paths(roots),
|
||||
key=lambda item: str(item).casefold(),
|
||||
)
|
||||
|
||||
for number, path in enumerate(paths, start=1):
|
||||
pdf = inspect_pdf(path, pdf_root)
|
||||
index[pdf.sha256].append(pdf)
|
||||
|
||||
print(
|
||||
f"\rPDFを走査中: {number}件",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if paths:
|
||||
print()
|
||||
|
||||
return dict(index)
|
||||
|
||||
|
||||
def open_database(
|
||||
database: Path,
|
||||
) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
|
||||
def load_papers(
|
||||
connection: sqlite3.Connection,
|
||||
) -> list[sqlite3.Row]:
|
||||
sql = """
|
||||
SELECT id, title, pdf_path, file_sha256
|
||||
FROM papers
|
||||
WHERE file_sha256 IS NOT NULL
|
||||
AND TRIM(file_sha256) <> ''
|
||||
ORDER BY id
|
||||
"""
|
||||
return connection.execute(sql).fetchall()
|
||||
|
||||
|
||||
def current_path_is_valid(
|
||||
pdf_root: Path,
|
||||
row: sqlite3.Row,
|
||||
) -> bool:
|
||||
if not row["pdf_path"]:
|
||||
return False
|
||||
|
||||
try:
|
||||
path = resolve_pdf_path(
|
||||
pdf_root,
|
||||
row["pdf_path"],
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if not path.is_file():
|
||||
return False
|
||||
|
||||
return (
|
||||
calculate_sha256(path)
|
||||
== row["file_sha256"].strip().lower()
|
||||
)
|
||||
|
||||
|
||||
def find_updates(
|
||||
rows: Iterable[sqlite3.Row],
|
||||
index: dict[str, list[PdfFile]],
|
||||
pdf_root: Path,
|
||||
) -> tuple[
|
||||
list[PathUpdate],
|
||||
list[sqlite3.Row],
|
||||
list[sqlite3.Row],
|
||||
]:
|
||||
updates: list[PathUpdate] = []
|
||||
ambiguous: list[sqlite3.Row] = []
|
||||
not_found: list[sqlite3.Row] = []
|
||||
|
||||
for row in rows:
|
||||
if current_path_is_valid(pdf_root, row):
|
||||
continue
|
||||
|
||||
sha256 = row["file_sha256"].strip().lower()
|
||||
candidates = index.get(sha256, [])
|
||||
|
||||
if not candidates:
|
||||
not_found.append(row)
|
||||
continue
|
||||
|
||||
if len(candidates) != 1:
|
||||
ambiguous.append(row)
|
||||
continue
|
||||
|
||||
candidate = candidates[0]
|
||||
|
||||
updates.append(
|
||||
PathUpdate(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
old_path=row["pdf_path"] or "",
|
||||
new_path=candidate.relative_path,
|
||||
sha256=sha256,
|
||||
file_size=candidate.file_size,
|
||||
file_mtime=candidate.file_mtime,
|
||||
)
|
||||
)
|
||||
|
||||
return updates, ambiguous, not_found
|
||||
|
||||
|
||||
def print_updates(
|
||||
updates: Iterable[PathUpdate],
|
||||
) -> None:
|
||||
items = list(updates)
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
print()
|
||||
print("=== 更新候補 ===")
|
||||
|
||||
for item in items:
|
||||
print(f"id: {item.paper_id}")
|
||||
print(f"title: {item.title}")
|
||||
print(f"old: {item.old_path or '(empty)'}")
|
||||
print(f"new: {item.new_path}")
|
||||
print()
|
||||
|
||||
|
||||
def print_rows(
|
||||
title: str,
|
||||
rows: Iterable[sqlite3.Row],
|
||||
) -> None:
|
||||
items = list(rows)
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
print()
|
||||
print(f"=== {title} ===")
|
||||
|
||||
for row in items:
|
||||
print(
|
||||
f"id={row['id']}: "
|
||||
f"{row['pdf_path']} "
|
||||
f"({row['title']})"
|
||||
)
|
||||
|
||||
|
||||
def create_backup(database: Path) -> Path:
|
||||
backup = database.with_name(
|
||||
database.name + ".before-path-update"
|
||||
)
|
||||
|
||||
counter = 1
|
||||
|
||||
while backup.exists():
|
||||
backup = database.with_name(
|
||||
database.name
|
||||
+ f".before-path-update.{counter}"
|
||||
)
|
||||
counter += 1
|
||||
|
||||
shutil.copy2(database, backup)
|
||||
return backup
|
||||
|
||||
|
||||
def apply_updates(
|
||||
connection: sqlite3.Connection,
|
||||
updates: Iterable[PathUpdate],
|
||||
) -> int:
|
||||
sql = """
|
||||
UPDATE papers
|
||||
SET pdf_path = ?,
|
||||
file_size = ?,
|
||||
file_mtime = ?
|
||||
WHERE id = ?
|
||||
AND lower(file_sha256) = ?
|
||||
"""
|
||||
|
||||
count = 0
|
||||
|
||||
with connection:
|
||||
for item in updates:
|
||||
cursor = connection.execute(
|
||||
sql,
|
||||
(
|
||||
item.new_path,
|
||||
item.file_size,
|
||||
item.file_mtime,
|
||||
item.paper_id,
|
||||
item.sha256,
|
||||
),
|
||||
)
|
||||
count += cursor.rowcount
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
try:
|
||||
database = args.database.expanduser().resolve()
|
||||
pdf_root = args.pdf_root.expanduser().resolve()
|
||||
|
||||
validate_paths(database, pdf_root)
|
||||
|
||||
index = build_sha_index(
|
||||
pdf_root,
|
||||
scan_roots(pdf_root, args.scan_all),
|
||||
)
|
||||
|
||||
with open_database(database) as connection:
|
||||
rows = load_papers(connection)
|
||||
updates, ambiguous, not_found = (
|
||||
find_updates(
|
||||
rows,
|
||||
index,
|
||||
pdf_root,
|
||||
)
|
||||
)
|
||||
|
||||
print_updates(updates)
|
||||
print_rows(
|
||||
"同一SHA256の候補が複数",
|
||||
ambiguous,
|
||||
)
|
||||
print_rows(
|
||||
"SHA256一致ファイルなし",
|
||||
not_found,
|
||||
)
|
||||
|
||||
if args.apply and updates:
|
||||
if not args.no_backup:
|
||||
backup = create_backup(database)
|
||||
print(f"DBバックアップ: {backup}")
|
||||
|
||||
changed = apply_updates(
|
||||
connection,
|
||||
updates,
|
||||
)
|
||||
print(f"DB更新: {changed} 件")
|
||||
|
||||
print()
|
||||
print("=== 概要 ===")
|
||||
print(
|
||||
"走査PDF: "
|
||||
f"{sum(len(v) for v in index.values())} 件"
|
||||
)
|
||||
print(f"DB照合対象: {len(rows)} 件")
|
||||
print(f"更新候補: {len(updates)} 件")
|
||||
print(f"曖昧な一致: {len(ambiguous)} 件")
|
||||
print(f"一致なし: {len(not_found)} 件")
|
||||
print(
|
||||
"実行モード: "
|
||||
+ ("更新実行" if args.apply else "dry-run")
|
||||
)
|
||||
|
||||
if updates and not args.apply:
|
||||
print()
|
||||
print(
|
||||
"更新するには --apply を指定してください。"
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"update_pdf_paths.py: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user