543 lines
15 KiB
Python
Executable File
543 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""tmp/paper-*.jsonをpapers.dbへ一括登録する。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import shutil
|
|
import sqlite3
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from common import (
|
|
ALLOWED_CATEGORIES,
|
|
ALLOWED_PRIORITIES,
|
|
CONFIG,
|
|
normalize_doi,
|
|
normalize_text,
|
|
now_iso,
|
|
resolve_pdf_path,
|
|
sha256_file,
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
"""コマンドライン引数を解析する。"""
|
|
parser = argparse.ArgumentParser(
|
|
description="tmp/paper-*.jsonをpapers.dbへ一括登録する"
|
|
)
|
|
parser.add_argument(
|
|
"--input-dir",
|
|
default=str(CONFIG.path.tmp_dir),
|
|
)
|
|
parser.add_argument(
|
|
"--database",
|
|
default=str(CONFIG.path.database),
|
|
)
|
|
parser.add_argument(
|
|
"--pdf-root",
|
|
default=str(CONFIG.path.pdf_root),
|
|
)
|
|
parser.add_argument(
|
|
"--pattern",
|
|
default="paper-*.json",
|
|
)
|
|
parser.add_argument(
|
|
"--continue-on-error",
|
|
action="store_true",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def backup_database(database: Path) -> Path:
|
|
"""DBを1回だけバックアップする。"""
|
|
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
backup = database.with_name(
|
|
f"{database.name}.bak-{stamp}"
|
|
)
|
|
shutil.copy2(database, backup)
|
|
return backup
|
|
|
|
|
|
def load_json(path: Path) -> dict[str, Any]:
|
|
"""論文JSONを読み込む。"""
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(data, dict):
|
|
raise ValueError("JSON root must be an object")
|
|
return data
|
|
|
|
|
|
def validate_json(data: dict[str, Any]) -> None:
|
|
"""JSONの必須項目と型を検証する。"""
|
|
required = ("title", "authors", "year", "journal", "pdf_path")
|
|
for key in required:
|
|
if key not in data:
|
|
raise ValueError(f"missing required field: {key}")
|
|
|
|
authors = data["authors"]
|
|
if not isinstance(authors, list) or not authors:
|
|
raise ValueError("authors must be a non-empty list")
|
|
|
|
relevance = int(data.get("relevance", 0))
|
|
if relevance not in range(6):
|
|
raise ValueError("relevance must be 0..5")
|
|
|
|
priority = data.get("priority", "D")
|
|
if priority not in ALLOWED_PRIORITIES:
|
|
raise ValueError("priority must be A..D")
|
|
|
|
category = data.get("category", "Other")
|
|
if category not in ALLOWED_CATEGORIES:
|
|
raise ValueError("unknown category")
|
|
|
|
|
|
def add_file_metadata(
|
|
data: dict[str, Any],
|
|
pdf_root: Path,
|
|
) -> None:
|
|
"""ローカルPDFの情報をJSONへ追加する。"""
|
|
actual_pdf = resolve_pdf_path(
|
|
pdf_root,
|
|
str(data["pdf_path"]),
|
|
)
|
|
if not actual_pdf.is_file():
|
|
raise FileNotFoundError(actual_pdf)
|
|
|
|
stat = actual_pdf.stat()
|
|
data["file_size"] = stat.st_size
|
|
data["file_mtime"] = datetime.fromtimestamp(
|
|
stat.st_mtime
|
|
).astimezone().isoformat(timespec="seconds")
|
|
data["file_sha256"] = sha256_file(actual_pdf)
|
|
|
|
|
|
def get_category_id(
|
|
connection: sqlite3.Connection,
|
|
name: str,
|
|
) -> int:
|
|
"""カテゴリIDを取得する。"""
|
|
connection.execute(
|
|
"INSERT OR IGNORE INTO categories(name) VALUES (?)",
|
|
(name,),
|
|
)
|
|
row = connection.execute(
|
|
"SELECT id FROM categories WHERE name = ?",
|
|
(name,),
|
|
).fetchone()
|
|
return int(row[0])
|
|
|
|
|
|
def get_author_id(
|
|
connection: sqlite3.Connection,
|
|
author: dict[str, Any],
|
|
) -> int:
|
|
"""著者を取得または登録する。"""
|
|
name = str(author.get("name", "")).strip()
|
|
normalized = normalize_text(name)
|
|
orcid = author.get("orcid")
|
|
|
|
row = connection.execute(
|
|
"""
|
|
SELECT id
|
|
FROM authors
|
|
WHERE name_normalized = ?
|
|
AND ((orcid = ?) OR (orcid IS NULL AND ? IS NULL))
|
|
ORDER BY id
|
|
LIMIT 1
|
|
""",
|
|
(normalized, orcid, orcid),
|
|
).fetchone()
|
|
|
|
if row:
|
|
connection.execute(
|
|
"""
|
|
UPDATE authors
|
|
SET affiliation = COALESCE(affiliation, ?),
|
|
orcid = COALESCE(orcid, ?)
|
|
WHERE id = ?
|
|
""",
|
|
(author.get("affiliation"), orcid, row[0]),
|
|
)
|
|
return int(row[0])
|
|
|
|
cursor = connection.execute(
|
|
"""
|
|
INSERT INTO authors(
|
|
name, name_normalized, orcid, affiliation
|
|
) VALUES (?, ?, ?, ?)
|
|
""",
|
|
(name, normalized, orcid, author.get("affiliation")),
|
|
)
|
|
return int(cursor.lastrowid)
|
|
|
|
|
|
def get_keyword_id(
|
|
connection: sqlite3.Connection,
|
|
keyword: Any,
|
|
) -> int:
|
|
"""キーワードIDを取得する。"""
|
|
name = " ".join(str(keyword).split())
|
|
normalized = normalize_text(name)
|
|
connection.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO keywords(name, name_normalized)
|
|
VALUES (?, ?)
|
|
""",
|
|
(name, normalized),
|
|
)
|
|
row = connection.execute(
|
|
"SELECT id FROM keywords WHERE name_normalized = ?",
|
|
(normalized,),
|
|
).fetchone()
|
|
return int(row[0])
|
|
|
|
|
|
def find_existing(
|
|
connection: sqlite3.Connection,
|
|
data: dict[str, Any],
|
|
) -> sqlite3.Row | None:
|
|
"""既存論文をSHA256、DOI、書誌情報の順で探す。"""
|
|
sha256 = data.get("file_sha256")
|
|
if sha256:
|
|
row = connection.execute(
|
|
"SELECT * FROM papers WHERE file_sha256 = ?",
|
|
(sha256,),
|
|
).fetchone()
|
|
if row:
|
|
return row
|
|
|
|
doi = normalize_doi(data.get("doi"))
|
|
if doi:
|
|
row = connection.execute(
|
|
"SELECT * FROM papers WHERE doi = ?",
|
|
(doi,),
|
|
).fetchone()
|
|
if row:
|
|
return row
|
|
|
|
first_author = (
|
|
data.get("first_author")
|
|
or data["authors"][0].get("name", "")
|
|
)
|
|
return connection.execute(
|
|
"""
|
|
SELECT *
|
|
FROM papers
|
|
WHERE title_normalized = ?
|
|
AND COALESCE(first_author, '') = ?
|
|
AND year IS ?
|
|
""",
|
|
(
|
|
normalize_text(str(data["title"])),
|
|
first_author,
|
|
data.get("year"),
|
|
),
|
|
).fetchone()
|
|
|
|
|
|
def update_existing_paper(
|
|
connection: sqlite3.Connection,
|
|
paper_id: int,
|
|
category_id: int,
|
|
data: dict[str, Any],
|
|
now: str,
|
|
) -> None:
|
|
"""既存論文の空欄を補完する。"""
|
|
connection.execute(
|
|
"""
|
|
UPDATE papers SET
|
|
doi = COALESCE(doi, ?),
|
|
journal = COALESCE(journal, ?),
|
|
volume = COALESCE(volume, ?),
|
|
issue = COALESCE(issue, ?),
|
|
pages = COALESCE(pages, ?),
|
|
publisher = COALESCE(publisher, ?),
|
|
article_type = COALESCE(article_type, ?),
|
|
language = COALESCE(language, ?),
|
|
abstract = COALESCE(abstract, ?),
|
|
summary_ja = COALESCE(summary_ja, ?),
|
|
category_id = COALESCE(category_id, ?),
|
|
pdf_path = COALESCE(pdf_path, ?),
|
|
file_size = COALESCE(file_size, ?),
|
|
file_mtime = COALESCE(file_mtime, ?),
|
|
file_sha256 = COALESCE(file_sha256, ?),
|
|
source = COALESCE(source, ?),
|
|
landing_url = COALESCE(landing_url, ?),
|
|
pdf_url = COALESCE(pdf_url, ?),
|
|
url_source = COALESCE(url_source, ?),
|
|
access_status = COALESCE(access_status, ?),
|
|
url_checked_at = COALESCE(url_checked_at, ?),
|
|
checked_at = ?,
|
|
updated_at = ?
|
|
WHERE id = ?
|
|
""",
|
|
(
|
|
normalize_doi(data.get("doi")),
|
|
data.get("journal"),
|
|
data.get("volume"),
|
|
data.get("issue"),
|
|
data.get("pages"),
|
|
data.get("publisher"),
|
|
data.get("article_type"),
|
|
data.get("language"),
|
|
data.get("abstract"),
|
|
data.get("summary_ja"),
|
|
category_id,
|
|
data.get("pdf_path"),
|
|
data["file_size"],
|
|
data["file_mtime"],
|
|
data["file_sha256"],
|
|
data.get("source"),
|
|
data.get("landing_url"),
|
|
data.get("pdf_url"),
|
|
data.get("url_source"),
|
|
data.get("access_status"),
|
|
data.get("url_checked_at"),
|
|
now,
|
|
now,
|
|
paper_id,
|
|
),
|
|
)
|
|
|
|
|
|
def insert_paper(
|
|
connection: sqlite3.Connection,
|
|
category_id: int,
|
|
data: dict[str, Any],
|
|
now: str,
|
|
) -> int:
|
|
"""新規論文を登録する。"""
|
|
first_author = (
|
|
data.get("first_author")
|
|
or data["authors"][0].get("name", "")
|
|
)
|
|
cursor = connection.execute(
|
|
"""
|
|
INSERT INTO papers(
|
|
doi, title, title_normalized, first_author, year, journal,
|
|
volume, issue, pages, publisher, article_type, language,
|
|
abstract, summary_ja, relevance, priority, review_status,
|
|
category_id, pdf_path, file_size, file_mtime, file_sha256,
|
|
source, landing_url, pdf_url, url_source, access_status,
|
|
url_checked_at, created_at, updated_at, checked_at, notes
|
|
) VALUES (
|
|
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
|
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
)
|
|
""",
|
|
(
|
|
normalize_doi(data.get("doi")),
|
|
data["title"],
|
|
normalize_text(str(data["title"])),
|
|
first_author,
|
|
data.get("year"),
|
|
data.get("journal"),
|
|
data.get("volume"),
|
|
data.get("issue"),
|
|
data.get("pages"),
|
|
data.get("publisher"),
|
|
data.get("article_type"),
|
|
data.get("language"),
|
|
data.get("abstract"),
|
|
data.get("summary_ja"),
|
|
int(data.get("relevance", 0)),
|
|
data.get("priority", "D"),
|
|
data.get("review_status", "unreviewed"),
|
|
category_id,
|
|
data.get("pdf_path"),
|
|
data["file_size"],
|
|
data["file_mtime"],
|
|
data["file_sha256"],
|
|
data.get("source"),
|
|
data.get("landing_url"),
|
|
data.get("pdf_url"),
|
|
data.get("url_source"),
|
|
data.get("access_status"),
|
|
data.get("url_checked_at"),
|
|
now,
|
|
now,
|
|
now,
|
|
data.get("notes"),
|
|
),
|
|
)
|
|
return int(cursor.lastrowid)
|
|
|
|
|
|
def register_authors(
|
|
connection: sqlite3.Connection,
|
|
paper_id: int,
|
|
authors: list[dict[str, Any]],
|
|
) -> None:
|
|
"""論文著者を登録する。"""
|
|
existing = connection.execute(
|
|
"SELECT 1 FROM paper_authors WHERE paper_id = ? LIMIT 1",
|
|
(paper_id,),
|
|
).fetchone()
|
|
if existing:
|
|
return
|
|
|
|
for order, author in enumerate(authors, start=1):
|
|
author_id = get_author_id(connection, author)
|
|
connection.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO paper_authors(
|
|
paper_id, author_id, author_order, is_corresponding
|
|
) VALUES (?, ?, ?, ?)
|
|
""",
|
|
(
|
|
paper_id,
|
|
author_id,
|
|
order,
|
|
int(bool(author.get("is_corresponding", False))),
|
|
),
|
|
)
|
|
|
|
|
|
def register_keywords(
|
|
connection: sqlite3.Connection,
|
|
paper_id: int,
|
|
keywords: list[Any],
|
|
) -> None:
|
|
"""論文キーワードを登録する。"""
|
|
for keyword in keywords:
|
|
keyword_id = get_keyword_id(connection, keyword)
|
|
connection.execute(
|
|
"""
|
|
INSERT OR IGNORE INTO paper_keywords(
|
|
paper_id, keyword_id
|
|
) VALUES (?, ?)
|
|
""",
|
|
(paper_id, keyword_id),
|
|
)
|
|
|
|
|
|
def register_one(
|
|
connection: sqlite3.Connection,
|
|
json_path: Path,
|
|
pdf_root: Path,
|
|
) -> dict[str, Any]:
|
|
"""1件のJSONを登録する。"""
|
|
data = load_json(json_path)
|
|
validate_json(data)
|
|
add_file_metadata(data, pdf_root)
|
|
|
|
connection.execute("BEGIN IMMEDIATE")
|
|
try:
|
|
existing = find_existing(connection, data)
|
|
category_id = get_category_id(
|
|
connection,
|
|
str(data.get("category", "Other")),
|
|
)
|
|
now = now_iso()
|
|
|
|
if existing:
|
|
paper_id = int(existing["id"])
|
|
update_existing_paper(
|
|
connection,
|
|
paper_id,
|
|
category_id,
|
|
data,
|
|
now,
|
|
)
|
|
status = "updated"
|
|
else:
|
|
paper_id = insert_paper(
|
|
connection,
|
|
category_id,
|
|
data,
|
|
now,
|
|
)
|
|
status = "registered"
|
|
|
|
register_authors(connection, paper_id, data["authors"])
|
|
register_keywords(
|
|
connection,
|
|
paper_id,
|
|
data.get("keywords", []),
|
|
)
|
|
connection.execute(
|
|
"""
|
|
INSERT INTO processing_log(
|
|
pdf_path, paper_id, status, message, processed_at
|
|
) VALUES (?, ?, ?, ?, ?)
|
|
""",
|
|
(data["pdf_path"], paper_id, status, status, now),
|
|
)
|
|
|
|
integrity = connection.execute(
|
|
"PRAGMA integrity_check"
|
|
).fetchone()[0]
|
|
if integrity != "ok":
|
|
raise RuntimeError(integrity)
|
|
|
|
connection.commit()
|
|
return {
|
|
"status": status,
|
|
"paper_id": paper_id,
|
|
"integrity_check": integrity,
|
|
}
|
|
except Exception:
|
|
connection.rollback()
|
|
raise
|
|
|
|
|
|
def process_files(args: argparse.Namespace) -> int:
|
|
"""対象JSONを一括処理する。"""
|
|
input_dir = Path(args.input_dir)
|
|
database = Path(args.database)
|
|
pdf_root = Path(args.pdf_root).expanduser().resolve()
|
|
files = sorted(input_dir.glob(args.pattern))
|
|
|
|
if not files:
|
|
print("新規JSONはありません。")
|
|
return 0
|
|
if not database.is_file():
|
|
raise FileNotFoundError(
|
|
f"database not found: {database}"
|
|
)
|
|
|
|
if CONFIG.processing.backup_database:
|
|
backup = backup_database(database)
|
|
print(f"Backup: {backup}")
|
|
|
|
counts = {"registered": 0, "updated": 0, "failed": 0}
|
|
with sqlite3.connect(database) as connection:
|
|
connection.row_factory = sqlite3.Row
|
|
connection.execute("PRAGMA foreign_keys=ON")
|
|
|
|
for path in files:
|
|
print(f"=== {path} ===")
|
|
try:
|
|
result = register_one(connection, path, pdf_root)
|
|
counts[result["status"]] += 1
|
|
print(json.dumps(result, ensure_ascii=False))
|
|
except Exception as exc:
|
|
counts["failed"] += 1
|
|
print(f"ERROR: {exc}", file=sys.stderr)
|
|
if not args.continue_on_error:
|
|
return 1
|
|
|
|
print()
|
|
print("=== 一括登録結果 ===")
|
|
print(f"対象: {len(files)}")
|
|
print(f"新規登録: {counts['registered']}")
|
|
print(f"既存更新: {counts['updated']}")
|
|
print(f"失敗: {counts['failed']}")
|
|
return 1 if counts["failed"] else 0
|
|
|
|
|
|
def main() -> int:
|
|
"""メイン処理。"""
|
|
try:
|
|
return process_files(parse_args())
|
|
except Exception as exc:
|
|
print(f"update_all.py: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|