Initial commit
This commit is contained in:
Executable
+542
@@ -0,0 +1,542 @@
|
||||
#!/usr/bin/env python3
|
||||
"""tmp/paper-*.jsonをpapers.dbへ一括登録する。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from common import (
|
||||
ALLOWED_CATEGORIES,
|
||||
ALLOWED_PRIORITIES,
|
||||
CONFIG,
|
||||
normalize_doi,
|
||||
normalize_text,
|
||||
now_iso,
|
||||
resolve_pdf_path,
|
||||
sha256_file,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""コマンドライン引数を解析する。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="tmp/paper-*.jsonをpapers.dbへ一括登録する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-dir",
|
||||
default=str(CONFIG.path.tmp_dir),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database",
|
||||
default=str(CONFIG.path.database),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pdf-root",
|
||||
default=str(CONFIG.path.pdf_root),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pattern",
|
||||
default="paper-*.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--continue-on-error",
|
||||
action="store_true",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def backup_database(database: Path) -> Path:
|
||||
"""DBを1回だけバックアップする。"""
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
backup = database.with_name(
|
||||
f"{database.name}.bak-{stamp}"
|
||||
)
|
||||
shutil.copy2(database, backup)
|
||||
return backup
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
"""論文JSONを読み込む。"""
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("JSON root must be an object")
|
||||
return data
|
||||
|
||||
|
||||
def validate_json(data: dict[str, Any]) -> None:
|
||||
"""JSONの必須項目と型を検証する。"""
|
||||
required = ("title", "authors", "year", "journal", "pdf_path")
|
||||
for key in required:
|
||||
if key not in data:
|
||||
raise ValueError(f"missing required field: {key}")
|
||||
|
||||
authors = data["authors"]
|
||||
if not isinstance(authors, list) or not authors:
|
||||
raise ValueError("authors must be a non-empty list")
|
||||
|
||||
relevance = int(data.get("relevance", 0))
|
||||
if relevance not in range(6):
|
||||
raise ValueError("relevance must be 0..5")
|
||||
|
||||
priority = data.get("priority", "D")
|
||||
if priority not in ALLOWED_PRIORITIES:
|
||||
raise ValueError("priority must be A..D")
|
||||
|
||||
category = data.get("category", "Other")
|
||||
if category not in ALLOWED_CATEGORIES:
|
||||
raise ValueError("unknown category")
|
||||
|
||||
|
||||
def add_file_metadata(
|
||||
data: dict[str, Any],
|
||||
pdf_root: Path,
|
||||
) -> None:
|
||||
"""ローカルPDFの情報をJSONへ追加する。"""
|
||||
actual_pdf = resolve_pdf_path(
|
||||
pdf_root,
|
||||
str(data["pdf_path"]),
|
||||
)
|
||||
if not actual_pdf.is_file():
|
||||
raise FileNotFoundError(actual_pdf)
|
||||
|
||||
stat = actual_pdf.stat()
|
||||
data["file_size"] = stat.st_size
|
||||
data["file_mtime"] = datetime.fromtimestamp(
|
||||
stat.st_mtime
|
||||
).astimezone().isoformat(timespec="seconds")
|
||||
data["file_sha256"] = sha256_file(actual_pdf)
|
||||
|
||||
|
||||
def get_category_id(
|
||||
connection: sqlite3.Connection,
|
||||
name: str,
|
||||
) -> int:
|
||||
"""カテゴリIDを取得する。"""
|
||||
connection.execute(
|
||||
"INSERT OR IGNORE INTO categories(name) VALUES (?)",
|
||||
(name,),
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT id FROM categories WHERE name = ?",
|
||||
(name,),
|
||||
).fetchone()
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def get_author_id(
|
||||
connection: sqlite3.Connection,
|
||||
author: dict[str, Any],
|
||||
) -> int:
|
||||
"""著者を取得または登録する。"""
|
||||
name = str(author.get("name", "")).strip()
|
||||
normalized = normalize_text(name)
|
||||
orcid = author.get("orcid")
|
||||
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM authors
|
||||
WHERE name_normalized = ?
|
||||
AND ((orcid = ?) OR (orcid IS NULL AND ? IS NULL))
|
||||
ORDER BY id
|
||||
LIMIT 1
|
||||
""",
|
||||
(normalized, orcid, orcid),
|
||||
).fetchone()
|
||||
|
||||
if row:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE authors
|
||||
SET affiliation = COALESCE(affiliation, ?),
|
||||
orcid = COALESCE(orcid, ?)
|
||||
WHERE id = ?
|
||||
""",
|
||||
(author.get("affiliation"), orcid, row[0]),
|
||||
)
|
||||
return int(row[0])
|
||||
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO authors(
|
||||
name, name_normalized, orcid, affiliation
|
||||
) VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(name, normalized, orcid, author.get("affiliation")),
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
|
||||
def get_keyword_id(
|
||||
connection: sqlite3.Connection,
|
||||
keyword: Any,
|
||||
) -> int:
|
||||
"""キーワードIDを取得する。"""
|
||||
name = " ".join(str(keyword).split())
|
||||
normalized = normalize_text(name)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO keywords(name, name_normalized)
|
||||
VALUES (?, ?)
|
||||
""",
|
||||
(name, normalized),
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT id FROM keywords WHERE name_normalized = ?",
|
||||
(normalized,),
|
||||
).fetchone()
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def find_existing(
|
||||
connection: sqlite3.Connection,
|
||||
data: dict[str, Any],
|
||||
) -> sqlite3.Row | None:
|
||||
"""既存論文をSHA256、DOI、書誌情報の順で探す。"""
|
||||
sha256 = data.get("file_sha256")
|
||||
if sha256:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM papers WHERE file_sha256 = ?",
|
||||
(sha256,),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row
|
||||
|
||||
doi = normalize_doi(data.get("doi"))
|
||||
if doi:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM papers WHERE doi = ?",
|
||||
(doi,),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row
|
||||
|
||||
first_author = (
|
||||
data.get("first_author")
|
||||
or data["authors"][0].get("name", "")
|
||||
)
|
||||
return connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM papers
|
||||
WHERE title_normalized = ?
|
||||
AND COALESCE(first_author, '') = ?
|
||||
AND year IS ?
|
||||
""",
|
||||
(
|
||||
normalize_text(str(data["title"])),
|
||||
first_author,
|
||||
data.get("year"),
|
||||
),
|
||||
).fetchone()
|
||||
|
||||
|
||||
def update_existing_paper(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
category_id: int,
|
||||
data: dict[str, Any],
|
||||
now: str,
|
||||
) -> None:
|
||||
"""既存論文の空欄を補完する。"""
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE papers SET
|
||||
doi = COALESCE(doi, ?),
|
||||
journal = COALESCE(journal, ?),
|
||||
volume = COALESCE(volume, ?),
|
||||
issue = COALESCE(issue, ?),
|
||||
pages = COALESCE(pages, ?),
|
||||
publisher = COALESCE(publisher, ?),
|
||||
article_type = COALESCE(article_type, ?),
|
||||
language = COALESCE(language, ?),
|
||||
abstract = COALESCE(abstract, ?),
|
||||
summary_ja = COALESCE(summary_ja, ?),
|
||||
category_id = COALESCE(category_id, ?),
|
||||
pdf_path = COALESCE(pdf_path, ?),
|
||||
file_size = COALESCE(file_size, ?),
|
||||
file_mtime = COALESCE(file_mtime, ?),
|
||||
file_sha256 = COALESCE(file_sha256, ?),
|
||||
source = COALESCE(source, ?),
|
||||
landing_url = COALESCE(landing_url, ?),
|
||||
pdf_url = COALESCE(pdf_url, ?),
|
||||
url_source = COALESCE(url_source, ?),
|
||||
access_status = COALESCE(access_status, ?),
|
||||
url_checked_at = COALESCE(url_checked_at, ?),
|
||||
checked_at = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
normalize_doi(data.get("doi")),
|
||||
data.get("journal"),
|
||||
data.get("volume"),
|
||||
data.get("issue"),
|
||||
data.get("pages"),
|
||||
data.get("publisher"),
|
||||
data.get("article_type"),
|
||||
data.get("language"),
|
||||
data.get("abstract"),
|
||||
data.get("summary_ja"),
|
||||
category_id,
|
||||
data.get("pdf_path"),
|
||||
data["file_size"],
|
||||
data["file_mtime"],
|
||||
data["file_sha256"],
|
||||
data.get("source"),
|
||||
data.get("landing_url"),
|
||||
data.get("pdf_url"),
|
||||
data.get("url_source"),
|
||||
data.get("access_status"),
|
||||
data.get("url_checked_at"),
|
||||
now,
|
||||
now,
|
||||
paper_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def insert_paper(
|
||||
connection: sqlite3.Connection,
|
||||
category_id: int,
|
||||
data: dict[str, Any],
|
||||
now: str,
|
||||
) -> int:
|
||||
"""新規論文を登録する。"""
|
||||
first_author = (
|
||||
data.get("first_author")
|
||||
or data["authors"][0].get("name", "")
|
||||
)
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO papers(
|
||||
doi, title, title_normalized, first_author, year, journal,
|
||||
volume, issue, pages, publisher, article_type, language,
|
||||
abstract, summary_ja, relevance, priority, review_status,
|
||||
category_id, pdf_path, file_size, file_mtime, file_sha256,
|
||||
source, landing_url, pdf_url, url_source, access_status,
|
||||
url_checked_at, created_at, updated_at, checked_at, notes
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
""",
|
||||
(
|
||||
normalize_doi(data.get("doi")),
|
||||
data["title"],
|
||||
normalize_text(str(data["title"])),
|
||||
first_author,
|
||||
data.get("year"),
|
||||
data.get("journal"),
|
||||
data.get("volume"),
|
||||
data.get("issue"),
|
||||
data.get("pages"),
|
||||
data.get("publisher"),
|
||||
data.get("article_type"),
|
||||
data.get("language"),
|
||||
data.get("abstract"),
|
||||
data.get("summary_ja"),
|
||||
int(data.get("relevance", 0)),
|
||||
data.get("priority", "D"),
|
||||
data.get("review_status", "unreviewed"),
|
||||
category_id,
|
||||
data.get("pdf_path"),
|
||||
data["file_size"],
|
||||
data["file_mtime"],
|
||||
data["file_sha256"],
|
||||
data.get("source"),
|
||||
data.get("landing_url"),
|
||||
data.get("pdf_url"),
|
||||
data.get("url_source"),
|
||||
data.get("access_status"),
|
||||
data.get("url_checked_at"),
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
data.get("notes"),
|
||||
),
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
|
||||
def register_authors(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
authors: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""論文著者を登録する。"""
|
||||
existing = connection.execute(
|
||||
"SELECT 1 FROM paper_authors WHERE paper_id = ? LIMIT 1",
|
||||
(paper_id,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
return
|
||||
|
||||
for order, author in enumerate(authors, start=1):
|
||||
author_id = get_author_id(connection, author)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO paper_authors(
|
||||
paper_id, author_id, author_order, is_corresponding
|
||||
) VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
paper_id,
|
||||
author_id,
|
||||
order,
|
||||
int(bool(author.get("is_corresponding", False))),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def register_keywords(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
keywords: list[Any],
|
||||
) -> None:
|
||||
"""論文キーワードを登録する。"""
|
||||
for keyword in keywords:
|
||||
keyword_id = get_keyword_id(connection, keyword)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO paper_keywords(
|
||||
paper_id, keyword_id
|
||||
) VALUES (?, ?)
|
||||
""",
|
||||
(paper_id, keyword_id),
|
||||
)
|
||||
|
||||
|
||||
def register_one(
|
||||
connection: sqlite3.Connection,
|
||||
json_path: Path,
|
||||
pdf_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""1件のJSONを登録する。"""
|
||||
data = load_json(json_path)
|
||||
validate_json(data)
|
||||
add_file_metadata(data, pdf_root)
|
||||
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
existing = find_existing(connection, data)
|
||||
category_id = get_category_id(
|
||||
connection,
|
||||
str(data.get("category", "Other")),
|
||||
)
|
||||
now = now_iso()
|
||||
|
||||
if existing:
|
||||
paper_id = int(existing["id"])
|
||||
update_existing_paper(
|
||||
connection,
|
||||
paper_id,
|
||||
category_id,
|
||||
data,
|
||||
now,
|
||||
)
|
||||
status = "updated"
|
||||
else:
|
||||
paper_id = insert_paper(
|
||||
connection,
|
||||
category_id,
|
||||
data,
|
||||
now,
|
||||
)
|
||||
status = "registered"
|
||||
|
||||
register_authors(connection, paper_id, data["authors"])
|
||||
register_keywords(
|
||||
connection,
|
||||
paper_id,
|
||||
data.get("keywords", []),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO processing_log(
|
||||
pdf_path, paper_id, status, message, processed_at
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(data["pdf_path"], paper_id, status, status, now),
|
||||
)
|
||||
|
||||
integrity = connection.execute(
|
||||
"PRAGMA integrity_check"
|
||||
).fetchone()[0]
|
||||
if integrity != "ok":
|
||||
raise RuntimeError(integrity)
|
||||
|
||||
connection.commit()
|
||||
return {
|
||||
"status": status,
|
||||
"paper_id": paper_id,
|
||||
"integrity_check": integrity,
|
||||
}
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def process_files(args: argparse.Namespace) -> int:
|
||||
"""対象JSONを一括処理する。"""
|
||||
input_dir = Path(args.input_dir)
|
||||
database = Path(args.database)
|
||||
pdf_root = Path(args.pdf_root).expanduser().resolve()
|
||||
files = sorted(input_dir.glob(args.pattern))
|
||||
|
||||
if not files:
|
||||
print("新規JSONはありません。")
|
||||
return 0
|
||||
if not database.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"database not found: {database}"
|
||||
)
|
||||
|
||||
if CONFIG.processing.backup_database:
|
||||
backup = backup_database(database)
|
||||
print(f"Backup: {backup}")
|
||||
|
||||
counts = {"registered": 0, "updated": 0, "failed": 0}
|
||||
with sqlite3.connect(database) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
|
||||
for path in files:
|
||||
print(f"=== {path} ===")
|
||||
try:
|
||||
result = register_one(connection, path, pdf_root)
|
||||
counts[result["status"]] += 1
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
except Exception as exc:
|
||||
counts["failed"] += 1
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
if not args.continue_on_error:
|
||||
return 1
|
||||
|
||||
print()
|
||||
print("=== 一括登録結果 ===")
|
||||
print(f"対象: {len(files)}")
|
||||
print(f"新規登録: {counts['registered']}")
|
||||
print(f"既存更新: {counts['updated']}")
|
||||
print(f"失敗: {counts['failed']}")
|
||||
return 1 if counts["failed"] else 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""メイン処理。"""
|
||||
try:
|
||||
return process_files(parse_args())
|
||||
except Exception as exc:
|
||||
print(f"update_all.py: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user