#!/usr/bin/env python3 """tmp/paper-*.jsonをpapers.dbへ一括登録する。""" from __future__ import annotations import argparse import json import shutil import sqlite3 import sys from datetime import datetime from pathlib import Path from typing import Any from common import ( ALLOWED_CATEGORIES, ALLOWED_PRIORITIES, CONFIG, normalize_doi, normalize_text, now_iso, resolve_pdf_path, sha256_file, ) def parse_args() -> argparse.Namespace: """コマンドライン引数を解析する。""" parser = argparse.ArgumentParser( description="tmp/paper-*.jsonをpapers.dbへ一括登録する" ) parser.add_argument( "--input-dir", default=str(CONFIG.path.tmp_dir), ) parser.add_argument( "--database", default=str(CONFIG.path.database), ) parser.add_argument( "--pdf-root", default=str(CONFIG.path.pdf_root), ) parser.add_argument( "--pattern", default="paper-*.json", ) parser.add_argument( "--continue-on-error", action="store_true", ) return parser.parse_args() def backup_database(database: Path) -> Path: """DBを1回だけバックアップする。""" stamp = datetime.now().strftime("%Y%m%d-%H%M%S") backup = database.with_name( f"{database.name}.bak-{stamp}" ) shutil.copy2(database, backup) return backup def load_json(path: Path) -> dict[str, Any]: """論文JSONを読み込む。""" data = json.loads(path.read_text(encoding="utf-8")) if not isinstance(data, dict): raise ValueError("JSON root must be an object") return data def validate_json(data: dict[str, Any]) -> None: """JSONの必須項目と型を検証する。""" required = ("title", "authors", "year", "journal", "pdf_path") for key in required: if key not in data: raise ValueError(f"missing required field: {key}") authors = data["authors"] if not isinstance(authors, list) or not authors: raise ValueError("authors must be a non-empty list") relevance = int(data.get("relevance", 0)) if relevance not in range(6): raise ValueError("relevance must be 0..5") priority = data.get("priority", "D") if priority not in ALLOWED_PRIORITIES: raise ValueError("priority must be A..D") category = data.get("category", "Other") if category not in ALLOWED_CATEGORIES: raise ValueError("unknown category") def add_file_metadata( data: dict[str, Any], pdf_root: Path, ) -> None: """ローカルPDFの情報をJSONへ追加する。""" actual_pdf = resolve_pdf_path( pdf_root, str(data["pdf_path"]), ) if not actual_pdf.is_file(): raise FileNotFoundError(actual_pdf) stat = actual_pdf.stat() data["file_size"] = stat.st_size data["file_mtime"] = datetime.fromtimestamp( stat.st_mtime ).astimezone().isoformat(timespec="seconds") data["file_sha256"] = sha256_file(actual_pdf) def get_category_id( connection: sqlite3.Connection, name: str, ) -> int: """カテゴリIDを取得する。""" connection.execute( "INSERT OR IGNORE INTO categories(name) VALUES (?)", (name,), ) row = connection.execute( "SELECT id FROM categories WHERE name = ?", (name,), ).fetchone() return int(row[0]) def get_author_id( connection: sqlite3.Connection, author: dict[str, Any], ) -> int: """著者を取得または登録する。""" name = str(author.get("name", "")).strip() normalized = normalize_text(name) orcid = author.get("orcid") row = connection.execute( """ SELECT id FROM authors WHERE name_normalized = ? AND ((orcid = ?) OR (orcid IS NULL AND ? IS NULL)) ORDER BY id LIMIT 1 """, (normalized, orcid, orcid), ).fetchone() if row: connection.execute( """ UPDATE authors SET affiliation = COALESCE(affiliation, ?), orcid = COALESCE(orcid, ?) WHERE id = ? """, (author.get("affiliation"), orcid, row[0]), ) return int(row[0]) cursor = connection.execute( """ INSERT INTO authors( name, name_normalized, orcid, affiliation ) VALUES (?, ?, ?, ?) """, (name, normalized, orcid, author.get("affiliation")), ) return int(cursor.lastrowid) def get_keyword_id( connection: sqlite3.Connection, keyword: Any, ) -> int: """キーワードIDを取得する。""" name = " ".join(str(keyword).split()) normalized = normalize_text(name) connection.execute( """ INSERT OR IGNORE INTO keywords(name, name_normalized) VALUES (?, ?) """, (name, normalized), ) row = connection.execute( "SELECT id FROM keywords WHERE name_normalized = ?", (normalized,), ).fetchone() return int(row[0]) def find_existing( connection: sqlite3.Connection, data: dict[str, Any], ) -> sqlite3.Row | None: """既存論文をSHA256、DOI、書誌情報の順で探す。""" sha256 = data.get("file_sha256") if sha256: row = connection.execute( "SELECT * FROM papers WHERE file_sha256 = ?", (sha256,), ).fetchone() if row: return row doi = normalize_doi(data.get("doi")) if doi: row = connection.execute( "SELECT * FROM papers WHERE doi = ?", (doi,), ).fetchone() if row: return row first_author = ( data.get("first_author") or data["authors"][0].get("name", "") ) return connection.execute( """ SELECT * FROM papers WHERE title_normalized = ? AND COALESCE(first_author, '') = ? AND year IS ? """, ( normalize_text(str(data["title"])), first_author, data.get("year"), ), ).fetchone() def update_existing_paper( connection: sqlite3.Connection, paper_id: int, category_id: int, data: dict[str, Any], now: str, ) -> None: """既存論文の空欄を補完する。""" connection.execute( """ UPDATE papers SET doi = COALESCE(doi, ?), journal = COALESCE(journal, ?), volume = COALESCE(volume, ?), issue = COALESCE(issue, ?), pages = COALESCE(pages, ?), publisher = COALESCE(publisher, ?), article_type = COALESCE(article_type, ?), language = COALESCE(language, ?), abstract = COALESCE(abstract, ?), summary_ja = COALESCE(summary_ja, ?), category_id = COALESCE(category_id, ?), pdf_path = COALESCE(pdf_path, ?), file_size = COALESCE(file_size, ?), file_mtime = COALESCE(file_mtime, ?), file_sha256 = COALESCE(file_sha256, ?), source = COALESCE(source, ?), landing_url = COALESCE(landing_url, ?), pdf_url = COALESCE(pdf_url, ?), url_source = COALESCE(url_source, ?), access_status = COALESCE(access_status, ?), url_checked_at = COALESCE(url_checked_at, ?), checked_at = ?, updated_at = ? WHERE id = ? """, ( normalize_doi(data.get("doi")), data.get("journal"), data.get("volume"), data.get("issue"), data.get("pages"), data.get("publisher"), data.get("article_type"), data.get("language"), data.get("abstract"), data.get("summary_ja"), category_id, data.get("pdf_path"), data["file_size"], data["file_mtime"], data["file_sha256"], data.get("source"), data.get("landing_url"), data.get("pdf_url"), data.get("url_source"), data.get("access_status"), data.get("url_checked_at"), now, now, paper_id, ), ) def insert_paper( connection: sqlite3.Connection, category_id: int, data: dict[str, Any], now: str, ) -> int: """新規論文を登録する。""" first_author = ( data.get("first_author") or data["authors"][0].get("name", "") ) cursor = connection.execute( """ INSERT INTO papers( doi, title, title_normalized, first_author, year, journal, volume, issue, pages, publisher, article_type, language, abstract, summary_ja, relevance, priority, review_status, category_id, pdf_path, file_size, file_mtime, file_sha256, source, landing_url, pdf_url, url_source, access_status, url_checked_at, created_at, updated_at, checked_at, notes ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) """, ( normalize_doi(data.get("doi")), data["title"], normalize_text(str(data["title"])), first_author, data.get("year"), data.get("journal"), data.get("volume"), data.get("issue"), data.get("pages"), data.get("publisher"), data.get("article_type"), data.get("language"), data.get("abstract"), data.get("summary_ja"), int(data.get("relevance", 0)), data.get("priority", "D"), data.get("review_status", "unreviewed"), category_id, data.get("pdf_path"), data["file_size"], data["file_mtime"], data["file_sha256"], data.get("source"), data.get("landing_url"), data.get("pdf_url"), data.get("url_source"), data.get("access_status"), data.get("url_checked_at"), now, now, now, data.get("notes"), ), ) return int(cursor.lastrowid) def register_authors( connection: sqlite3.Connection, paper_id: int, authors: list[dict[str, Any]], ) -> None: """論文著者を登録する。""" existing = connection.execute( "SELECT 1 FROM paper_authors WHERE paper_id = ? LIMIT 1", (paper_id,), ).fetchone() if existing: return for order, author in enumerate(authors, start=1): author_id = get_author_id(connection, author) connection.execute( """ INSERT OR IGNORE INTO paper_authors( paper_id, author_id, author_order, is_corresponding ) VALUES (?, ?, ?, ?) """, ( paper_id, author_id, order, int(bool(author.get("is_corresponding", False))), ), ) def register_keywords( connection: sqlite3.Connection, paper_id: int, keywords: list[Any], ) -> None: """論文キーワードを登録する。""" for keyword in keywords: keyword_id = get_keyword_id(connection, keyword) connection.execute( """ INSERT OR IGNORE INTO paper_keywords( paper_id, keyword_id ) VALUES (?, ?) """, (paper_id, keyword_id), ) def register_one( connection: sqlite3.Connection, json_path: Path, pdf_root: Path, ) -> dict[str, Any]: """1件のJSONを登録する。""" data = load_json(json_path) validate_json(data) add_file_metadata(data, pdf_root) connection.execute("BEGIN IMMEDIATE") try: existing = find_existing(connection, data) category_id = get_category_id( connection, str(data.get("category", "Other")), ) now = now_iso() if existing: paper_id = int(existing["id"]) update_existing_paper( connection, paper_id, category_id, data, now, ) status = "updated" else: paper_id = insert_paper( connection, category_id, data, now, ) status = "registered" register_authors(connection, paper_id, data["authors"]) register_keywords( connection, paper_id, data.get("keywords", []), ) connection.execute( """ INSERT INTO processing_log( pdf_path, paper_id, status, message, processed_at ) VALUES (?, ?, ?, ?, ?) """, (data["pdf_path"], paper_id, status, status, now), ) integrity = connection.execute( "PRAGMA integrity_check" ).fetchone()[0] if integrity != "ok": raise RuntimeError(integrity) connection.commit() return { "status": status, "paper_id": paper_id, "integrity_check": integrity, } except Exception: connection.rollback() raise def process_files(args: argparse.Namespace) -> int: """対象JSONを一括処理する。""" input_dir = Path(args.input_dir) database = Path(args.database) pdf_root = Path(args.pdf_root).expanduser().resolve() files = sorted(input_dir.glob(args.pattern)) if not files: print("新規JSONはありません。") return 0 if not database.is_file(): raise FileNotFoundError( f"database not found: {database}" ) if CONFIG.processing.backup_database: backup = backup_database(database) print(f"Backup: {backup}") counts = {"registered": 0, "updated": 0, "failed": 0} with sqlite3.connect(database) as connection: connection.row_factory = sqlite3.Row connection.execute("PRAGMA foreign_keys=ON") for path in files: print(f"=== {path} ===") try: result = register_one(connection, path, pdf_root) counts[result["status"]] += 1 print(json.dumps(result, ensure_ascii=False)) except Exception as exc: counts["failed"] += 1 print(f"ERROR: {exc}", file=sys.stderr) if not args.continue_on_error: return 1 print() print("=== 一括登録結果 ===") print(f"対象: {len(files)}") print(f"新規登録: {counts['registered']}") print(f"既存更新: {counts['updated']}") print(f"失敗: {counts['failed']}") return 1 if counts["failed"] else 0 def main() -> int: """メイン処理。""" try: return process_files(parse_args()) except Exception as exc: print(f"update_all.py: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())