Initial commit
This commit is contained in:
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import sys
|
||||
|
||||
COMMON_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(COMMON_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(COMMON_ROOT))
|
||||
|
||||
from project_loader import load_config # noqa: E402
|
||||
|
||||
_, CONFIG = load_config()
|
||||
ALLOWED_PRIORITIES = set(CONFIG.domain.priorities)
|
||||
ALLOWED_CATEGORIES = set(CONFIG.domain.categories)
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
def normalize_doi(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
doi = str(value).strip().lower()
|
||||
if not doi or doi == "unknown":
|
||||
return None
|
||||
prefixes = (
|
||||
"https://doi.org/",
|
||||
"http://doi.org/",
|
||||
"http://dx.doi.org/",
|
||||
"doi:",
|
||||
)
|
||||
for prefix in prefixes:
|
||||
if doi.startswith(prefix):
|
||||
doi = doi[len(prefix):]
|
||||
break
|
||||
doi = doi.strip().rstrip(".,;:)]}")
|
||||
return doi or None
|
||||
|
||||
def normalize_text(value: str) -> str:
|
||||
value = html.unescape(value)
|
||||
value = unicodedata.normalize("NFKC", value)
|
||||
value = value.lower()
|
||||
value = re.sub(r"[\r\n\t]+", " ", value)
|
||||
value = re.sub(r"[^\w\s]", " ", value, flags=re.UNICODE)
|
||||
value = re.sub(r"\s+", " ", value).strip()
|
||||
return value
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for block in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
def resolve_pdf_path(pdf_root: Path, stored_path: str) -> Path:
|
||||
path = Path(stored_path)
|
||||
if path.is_absolute():
|
||||
raise ValueError("pdf_path must be relative to PDF_ROOT")
|
||||
resolved = (pdf_root / path).resolve()
|
||||
root = pdf_root.resolve()
|
||||
try:
|
||||
resolved.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise ValueError("pdf_path escapes PDF_ROOT") from exc
|
||||
return resolved
|
||||
Executable
+269
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
"""paper_idを指定してpapers.dbから論文を削除する。
|
||||
|
||||
PDFファイルそのものは削除しない。
|
||||
既定はdry-run。実際に削除する場合は--applyを指定する。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Paper:
|
||||
paper_id: int
|
||||
title: str
|
||||
first_author: str | None
|
||||
year: int | None
|
||||
doi: str | None
|
||||
pdf_path: str | None
|
||||
file_sha256: str | None
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="paper_idを指定してDBレコードを削除する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"paper_id",
|
||||
type=int,
|
||||
help="削除するpapers.id",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database",
|
||||
type=Path,
|
||||
default=CONFIG.path.database,
|
||||
help="SQLiteデータベース",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="削除を実際にDBへ反映する",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-backup",
|
||||
action="store_true",
|
||||
help="削除前バックアップを作成しない",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def open_database(
|
||||
database: Path,
|
||||
) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
|
||||
def load_paper(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
) -> Paper | None:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
id,
|
||||
title,
|
||||
first_author,
|
||||
year,
|
||||
doi,
|
||||
pdf_path,
|
||||
file_sha256
|
||||
FROM papers
|
||||
WHERE id = ?
|
||||
""",
|
||||
(paper_id,),
|
||||
).fetchone()
|
||||
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
return Paper(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
first_author=row["first_author"],
|
||||
year=row["year"],
|
||||
doi=row["doi"],
|
||||
pdf_path=row["pdf_path"],
|
||||
file_sha256=row["file_sha256"],
|
||||
)
|
||||
|
||||
|
||||
def print_paper(paper: Paper) -> None:
|
||||
print("=== 削除対象 ===")
|
||||
print(f"id: {paper.paper_id}")
|
||||
print(f"title: {paper.title}")
|
||||
print(
|
||||
"first_author: "
|
||||
f"{paper.first_author or '(none)'}"
|
||||
)
|
||||
print(f"year: {paper.year or '(none)'}")
|
||||
print(f"doi: {paper.doi or '(none)'}")
|
||||
print(
|
||||
f"pdf_path: {paper.pdf_path or '(none)'}"
|
||||
)
|
||||
print(
|
||||
"file_sha256: "
|
||||
f"{paper.file_sha256 or '(none)'}"
|
||||
)
|
||||
|
||||
|
||||
def create_backup(database: Path) -> Path:
|
||||
backup = database.with_name(
|
||||
database.name + ".before-delete"
|
||||
)
|
||||
|
||||
counter = 1
|
||||
|
||||
while backup.exists():
|
||||
backup = database.with_name(
|
||||
database.name + f".before-delete.{counter}"
|
||||
)
|
||||
counter += 1
|
||||
|
||||
shutil.copy2(database, backup)
|
||||
return backup
|
||||
|
||||
|
||||
def delete_paper(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
) -> None:
|
||||
"""関連レコードを含めて1トランザクションで削除する。"""
|
||||
with connection:
|
||||
tables = (
|
||||
"paper_authors",
|
||||
"paper_keywords",
|
||||
"processing_log",
|
||||
)
|
||||
|
||||
for table in tables:
|
||||
if table_exists(connection, table):
|
||||
connection.execute(
|
||||
f"DELETE FROM {table} WHERE paper_id = ?",
|
||||
(paper_id,),
|
||||
)
|
||||
|
||||
cursor = connection.execute(
|
||||
"DELETE FROM papers WHERE id = ?",
|
||||
(paper_id,),
|
||||
)
|
||||
|
||||
if cursor.rowcount != 1:
|
||||
raise RuntimeError(
|
||||
f"papers.id={paper_id}を削除できませんでした"
|
||||
)
|
||||
|
||||
|
||||
def table_exists(
|
||||
connection: sqlite3.Connection,
|
||||
table: str,
|
||||
) -> bool:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM sqlite_master
|
||||
WHERE type = 'table'
|
||||
AND name = ?
|
||||
""",
|
||||
(table,),
|
||||
).fetchone()
|
||||
|
||||
return row is not None
|
||||
|
||||
|
||||
def validate_database(
|
||||
connection: sqlite3.Connection,
|
||||
) -> None:
|
||||
integrity = connection.execute(
|
||||
"PRAGMA integrity_check"
|
||||
).fetchone()[0]
|
||||
|
||||
if integrity != "ok":
|
||||
raise RuntimeError(
|
||||
f"integrity_check: {integrity}"
|
||||
)
|
||||
|
||||
foreign_keys = connection.execute(
|
||||
"PRAGMA foreign_key_check"
|
||||
).fetchall()
|
||||
|
||||
if foreign_keys:
|
||||
raise RuntimeError(
|
||||
"foreign_key_check: "
|
||||
f"{len(foreign_keys)}件"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
try:
|
||||
database = args.database.expanduser().resolve()
|
||||
|
||||
if not database.is_file():
|
||||
raise FileNotFoundError(database)
|
||||
|
||||
with open_database(database) as connection:
|
||||
paper = load_paper(
|
||||
connection,
|
||||
args.paper_id,
|
||||
)
|
||||
|
||||
if paper is None:
|
||||
print(
|
||||
f"paper_id={args.paper_id}は存在しません。",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
print_paper(paper)
|
||||
|
||||
if not args.apply:
|
||||
print()
|
||||
print(
|
||||
"削除するには --apply を指定してください。"
|
||||
)
|
||||
return 0
|
||||
|
||||
if not args.no_backup:
|
||||
backup = create_backup(database)
|
||||
print()
|
||||
print(f"DBバックアップ: {backup}")
|
||||
|
||||
delete_paper(
|
||||
connection,
|
||||
args.paper_id,
|
||||
)
|
||||
validate_database(connection)
|
||||
|
||||
print()
|
||||
print(
|
||||
f"Deleted paper_id={args.paper_id}"
|
||||
)
|
||||
print("PDFファイルは削除していません。")
|
||||
return 0
|
||||
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"delete_paper.py: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+205
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Sequence, TextIO
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
|
||||
def prop(value: object) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return str(value).replace("\n", " ").strip()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"database",
|
||||
nargs="?",
|
||||
default=str(CONFIG.path.database),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
help="出力先。省略時は標準出力",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def open_database(database: str) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
return connection
|
||||
|
||||
|
||||
def load_papers(
|
||||
connection: sqlite3.Connection,
|
||||
) -> list[sqlite3.Row]:
|
||||
query = """
|
||||
SELECT p.*, c.name AS category
|
||||
FROM papers AS p
|
||||
LEFT JOIN categories AS c
|
||||
ON c.id = p.category_id
|
||||
ORDER BY
|
||||
COALESCE(p.year, 0) DESC,
|
||||
p.first_author,
|
||||
p.title
|
||||
"""
|
||||
return connection.execute(query).fetchall()
|
||||
|
||||
|
||||
def load_authors(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
) -> list[str]:
|
||||
query = """
|
||||
SELECT a.name
|
||||
FROM paper_authors AS pa
|
||||
JOIN authors AS a
|
||||
ON a.id = pa.author_id
|
||||
WHERE pa.paper_id = ?
|
||||
ORDER BY pa.author_order
|
||||
"""
|
||||
rows = connection.execute(query, (paper_id,)).fetchall()
|
||||
return [row["name"] for row in rows]
|
||||
|
||||
|
||||
def load_keywords(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
) -> list[str]:
|
||||
query = """
|
||||
SELECT k.name
|
||||
FROM paper_keywords AS pk
|
||||
JOIN keywords AS k
|
||||
ON k.id = pk.keyword_id
|
||||
WHERE pk.paper_id = ?
|
||||
ORDER BY k.name_normalized
|
||||
"""
|
||||
rows = connection.execute(query, (paper_id,)).fetchall()
|
||||
return [row["name"] for row in rows]
|
||||
|
||||
|
||||
def make_header() -> list[str]:
|
||||
exported_at = (
|
||||
datetime.now(timezone.utc)
|
||||
.astimezone()
|
||||
.isoformat(timespec="seconds")
|
||||
)
|
||||
return [
|
||||
"#+TITLE: ガラス固化体論文データベース",
|
||||
f"#+DB_EXPORTED_AT: {exported_at}",
|
||||
"",
|
||||
(
|
||||
"# 編集可能: CATEGORY, RELEVANCE, PRIORITY, STATUS, "
|
||||
"日本語要約, キーワード, メモ"
|
||||
),
|
||||
(
|
||||
"# 保護対象: DB_ID, DOI, TITLE, AUTHORS, YEAR, "
|
||||
"JOURNAL, PDF_PATH, FILE_SHA256"
|
||||
),
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def make_properties(
|
||||
paper: sqlite3.Row,
|
||||
authors: Sequence[str],
|
||||
) -> list[str]:
|
||||
category = paper["category"] or "Other"
|
||||
return [
|
||||
":PROPERTIES:",
|
||||
f":DB_ID: {paper['id']}",
|
||||
f":DB_UPDATED_AT: {prop(paper['updated_at'])}",
|
||||
f":DOI: {prop(paper['doi'])}",
|
||||
f":TITLE: {prop(paper['title'])}",
|
||||
f":AUTHORS: {prop('; '.join(authors))}",
|
||||
f":FIRST_AUTHOR: {prop(paper['first_author'])}",
|
||||
f":YEAR: {prop(paper['year'])}",
|
||||
f":JOURNAL: {prop(paper['journal'])}",
|
||||
f":PDF_PATH: {prop(paper['pdf_path'])}",
|
||||
f":FILE_SHA256: {prop(paper['file_sha256'])}",
|
||||
f":CATEGORY: {prop(category)}",
|
||||
f":RELEVANCE: {prop(paper['relevance'])}",
|
||||
f":PRIORITY: {prop(paper['priority'])}",
|
||||
f":STATUS: {prop(paper['review_status'])}",
|
||||
":END:",
|
||||
]
|
||||
|
||||
|
||||
def make_paper_entry(
|
||||
paper: sqlite3.Row,
|
||||
authors: Sequence[str],
|
||||
keywords: Sequence[str],
|
||||
) -> list[str]:
|
||||
paper_id = paper["id"]
|
||||
title = prop(paper["title"])
|
||||
lines = [
|
||||
f"* [ID:{paper_id}] {title}",
|
||||
*make_properties(paper, authors),
|
||||
"",
|
||||
"** 日本語要約",
|
||||
prop(paper["summary_ja"]),
|
||||
"",
|
||||
"** キーワード",
|
||||
]
|
||||
lines.extend(f"- {keyword}" for keyword in keywords)
|
||||
lines.extend(["", "** メモ", prop(paper["notes"]), ""])
|
||||
return lines
|
||||
|
||||
|
||||
def build_org(
|
||||
connection: sqlite3.Connection,
|
||||
papers: Sequence[sqlite3.Row],
|
||||
) -> str:
|
||||
lines = make_header()
|
||||
for paper in papers:
|
||||
paper_id = paper["id"]
|
||||
authors = load_authors(connection, paper_id)
|
||||
keywords = load_keywords(connection, paper_id)
|
||||
lines.extend(make_paper_entry(paper, authors, keywords))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_output(content: str, output: str | None) -> None:
|
||||
if output is None:
|
||||
sys.stdout.write(content)
|
||||
if not content.endswith("\n"):
|
||||
sys.stdout.write("\n")
|
||||
return
|
||||
|
||||
Path(output).write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def print_summary(
|
||||
paper_count: int,
|
||||
output: str | None,
|
||||
stream: TextIO = sys.stderr,
|
||||
) -> None:
|
||||
destination = output if output is not None else "stdout"
|
||||
print(
|
||||
f"Exported {paper_count} papers to {destination}",
|
||||
file=stream,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
with open_database(args.database) as connection:
|
||||
papers = load_papers(connection)
|
||||
content = build_org(connection, papers)
|
||||
|
||||
write_output(content, args.output)
|
||||
print_summary(len(papers), args.output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+1137
File diff suppressed because it is too large
Load Diff
Executable
+161
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
CATEGORIES = CONFIG.domain.categories
|
||||
|
||||
SCHEMA = """
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS categories (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS papers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
doi TEXT,
|
||||
title TEXT NOT NULL,
|
||||
title_normalized TEXT NOT NULL,
|
||||
first_author TEXT,
|
||||
year INTEGER,
|
||||
journal TEXT,
|
||||
volume TEXT,
|
||||
issue TEXT,
|
||||
pages TEXT,
|
||||
publisher TEXT,
|
||||
article_type TEXT,
|
||||
language TEXT,
|
||||
abstract TEXT,
|
||||
summary_ja TEXT,
|
||||
relevance INTEGER NOT NULL DEFAULT 0 CHECK (relevance BETWEEN 0 AND 5),
|
||||
priority TEXT NOT NULL DEFAULT 'D' CHECK (priority IN ('A','B','C','D')),
|
||||
review_status TEXT NOT NULL DEFAULT 'unreviewed',
|
||||
category_id INTEGER,
|
||||
pdf_path TEXT,
|
||||
file_size INTEGER,
|
||||
file_mtime TEXT,
|
||||
file_sha256 TEXT,
|
||||
source TEXT,
|
||||
landing_url TEXT,
|
||||
pdf_url TEXT,
|
||||
url_source TEXT,
|
||||
access_status TEXT,
|
||||
url_checked_at TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
checked_at TEXT,
|
||||
notes TEXT,
|
||||
FOREIGN KEY (category_id) REFERENCES categories(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS authors (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
name_normalized TEXT NOT NULL,
|
||||
orcid TEXT,
|
||||
affiliation TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS paper_authors (
|
||||
paper_id INTEGER NOT NULL,
|
||||
author_id INTEGER NOT NULL,
|
||||
author_order INTEGER NOT NULL,
|
||||
is_corresponding INTEGER NOT NULL DEFAULT 0 CHECK (is_corresponding IN (0,1)),
|
||||
PRIMARY KEY (paper_id, author_order),
|
||||
UNIQUE (paper_id, author_id),
|
||||
FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (author_id) REFERENCES authors(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS keywords (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
name_normalized TEXT NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS paper_keywords (
|
||||
paper_id INTEGER NOT NULL,
|
||||
keyword_id INTEGER NOT NULL,
|
||||
PRIMARY KEY (paper_id, keyword_id),
|
||||
FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (keyword_id) REFERENCES keywords(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS processing_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
pdf_path TEXT,
|
||||
paper_id INTEGER,
|
||||
status TEXT NOT NULL CHECK (status IN ('registered','updated','duplicate','skipped','error')),
|
||||
message TEXT,
|
||||
processed_at TEXT NOT NULL,
|
||||
FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_papers_doi
|
||||
ON papers(doi) WHERE doi IS NOT NULL AND doi <> '';
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_papers_sha256
|
||||
ON papers(file_sha256) WHERE file_sha256 IS NOT NULL AND file_sha256 <> '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_title_normalized ON papers(title_normalized);
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_year ON papers(year);
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_relevance ON papers(relevance);
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_priority ON papers(priority);
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_category_id ON papers(category_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_authors_name_normalized ON authors(name_normalized);
|
||||
CREATE INDEX IF NOT EXISTS idx_keywords_name_normalized ON keywords(name_normalized);
|
||||
"""
|
||||
|
||||
URL_COLUMNS = {
|
||||
"landing_url": "TEXT",
|
||||
"pdf_url": "TEXT",
|
||||
"url_source": "TEXT",
|
||||
"access_status": "TEXT",
|
||||
"url_checked_at": "TEXT",
|
||||
}
|
||||
|
||||
|
||||
def ensure_url_columns(con: sqlite3.Connection) -> None:
|
||||
columns = {
|
||||
row[1] for row in con.execute("PRAGMA table_info(papers)")
|
||||
}
|
||||
for name, column_type in URL_COLUMNS.items():
|
||||
if name not in columns:
|
||||
con.execute(
|
||||
f'ALTER TABLE papers ADD COLUMN "{name}" {column_type}'
|
||||
)
|
||||
|
||||
|
||||
def initialize(path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with sqlite3.connect(path) as con:
|
||||
con.executescript(SCHEMA)
|
||||
ensure_url_columns(con)
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO categories(name, description) VALUES (?, NULL)",
|
||||
[(name,) for name in CATEGORIES],
|
||||
)
|
||||
result = con.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if result != "ok":
|
||||
raise RuntimeError(result)
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument(
|
||||
"database", nargs="?", default=str(CONFIG.path.database)
|
||||
)
|
||||
args = p.parse_args()
|
||||
initialize(Path(args.database))
|
||||
print(f"Initialized: {args.database}")
|
||||
print("integrity_check: ok")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""config.py の設定値を表示する互換ユーティリティ。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
data = {
|
||||
"PDF_ROOT": str(CONFIG.path.pdf_root),
|
||||
"SCAN_DIRECTORIES": list(
|
||||
CONFIG.processing.scan_directories
|
||||
),
|
||||
"REMOTE": CONFIG.remote.host,
|
||||
"REMOTE_DB": CONFIG.remote.database_dir,
|
||||
"REMOTE_PDF": CONFIG.remote.pdf_dir,
|
||||
"MAX_PAPERS": CONFIG.processing.max_papers,
|
||||
"AI_BACKEND": CONFIG.ai.backend,
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
for key, value in data.items():
|
||||
print(f"{key}={value}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+220
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""設定に応じてClaude CodeまたはCodex CLIを実行する。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
|
||||
COMMON_ROOT = Path(__file__).resolve().parents[1]
|
||||
INSTRUCTIONS_FILE = COMMON_ROOT / "CLAUDE.md"
|
||||
|
||||
AI_PROMPT = """
|
||||
以下の共通指示に従い、TASK=process_new_papersを
|
||||
追加確認なしで最後まで実行してください。
|
||||
|
||||
--- 共通指示開始 ---
|
||||
{common_instructions}
|
||||
--- 共通指示終了 ---
|
||||
|
||||
tmp/text/には、未登録PDFから抽出したテキストがあります。
|
||||
|
||||
対象分野:
|
||||
{domain_name}
|
||||
|
||||
対象分野の説明:
|
||||
{domain_description}
|
||||
|
||||
categoryは次のいずれかを使用してください。
|
||||
{categories}
|
||||
|
||||
日本語要約は{summary_length_ja}を目安とし、
|
||||
次の観点を優先してください。
|
||||
{summary_focus}
|
||||
|
||||
relevanceは次の基準で判定してください。
|
||||
{relevance_criteria}
|
||||
|
||||
Priorityは次の対応にしてください。
|
||||
- A: relevance=5
|
||||
- B: relevance=4
|
||||
- C: relevance=3
|
||||
- D: relevance=0〜2
|
||||
|
||||
行う処理は次だけです。
|
||||
|
||||
- tmp/text/にあるテキストファイルを解析する
|
||||
- 各テキスト先頭のPDF_PATHをJSONのpdf_pathに使う
|
||||
- DOI、書誌情報、要約、キーワード、分類を抽出する
|
||||
- DOIがある場合はlanding_urlを生成する
|
||||
- 必要最小限のWeb検索で公開PDFを確認する
|
||||
- 各論文についてtmp/paper-*.jsonを作成する
|
||||
- 作成件数と結果を日本語で報告する
|
||||
|
||||
原則としてPDFは直接読まないでください。
|
||||
|
||||
次の場合のみPDF_PATHに記載された元PDFを参照できます。
|
||||
|
||||
- PDF_TEXT_EMPTYがyes
|
||||
- TEXT_INSUFFICIENTがyes
|
||||
- TEXT_LENGTHが{min_text_length}未満
|
||||
- テキストが文字化けしている
|
||||
- タイトル、著者、発行年、要約を確認できない
|
||||
|
||||
元PDFを参照する場合も、PDF_PATHに記載された1ファイルだけを読み、
|
||||
他のPDFやディレクトリを探索しないでください。
|
||||
|
||||
確認できない内容、DOI、pdf_urlは推測しないでください。
|
||||
papers.dbの直接更新、DB検証、Org出力は実行しないでください。
|
||||
最大{max_papers}件だけ処理してください。
|
||||
""".strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeOptions:
|
||||
backend: str
|
||||
max_papers: int
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
number = int(value)
|
||||
if number < 1:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"1以上の整数を指定してください"
|
||||
)
|
||||
return number
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Claude CodeまたはCodex CLIを実行する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
choices=("claude", "codex"),
|
||||
default=None,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-papers",
|
||||
type=positive_int,
|
||||
default=None,
|
||||
metavar="N",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def resolve_options(args: argparse.Namespace) -> RuntimeOptions:
|
||||
return RuntimeOptions(
|
||||
backend=args.backend or CONFIG.ai.backend,
|
||||
max_papers=(
|
||||
args.max_papers
|
||||
if args.max_papers is not None
|
||||
else CONFIG.processing.max_papers
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def load_common_instructions() -> str:
|
||||
if not INSTRUCTIONS_FILE.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"共通CLAUDE.mdがありません: {INSTRUCTIONS_FILE}"
|
||||
)
|
||||
return INSTRUCTIONS_FILE.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def build_prompt(max_papers: int) -> str:
|
||||
domain = CONFIG.domain
|
||||
return AI_PROMPT.format(
|
||||
common_instructions=load_common_instructions(),
|
||||
domain_name=domain.name,
|
||||
domain_description=domain.description,
|
||||
categories=domain.categories_prompt(),
|
||||
summary_length_ja=domain.summary_length_ja,
|
||||
summary_focus=domain.summary_focus_prompt(),
|
||||
relevance_criteria=domain.relevance_prompt(),
|
||||
min_text_length=CONFIG.processing.min_text_length,
|
||||
max_papers=max_papers,
|
||||
)
|
||||
|
||||
|
||||
def build_claude_command() -> list[str]:
|
||||
command = [
|
||||
"claude",
|
||||
"--print",
|
||||
"--add-dir",
|
||||
str(CONFIG.path.text_dir),
|
||||
"--permission-mode",
|
||||
CONFIG.ai.claude_permission_mode,
|
||||
"--allowedTools",
|
||||
CONFIG.ai.claude_allowed_tools_argument,
|
||||
]
|
||||
for path in CONFIG.scan_paths:
|
||||
command.extend(["--add-dir", str(path)])
|
||||
return command
|
||||
|
||||
|
||||
def build_codex_command(max_papers: int) -> list[str]:
|
||||
command = [
|
||||
"codex",
|
||||
"exec",
|
||||
"--cd",
|
||||
str(CONFIG.path.project_root),
|
||||
"--sandbox",
|
||||
CONFIG.ai.codex_sandbox,
|
||||
]
|
||||
if CONFIG.ai.codex_model:
|
||||
command.extend(["--model", CONFIG.ai.codex_model])
|
||||
if CONFIG.ai.codex_full_auto:
|
||||
command.append("--full-auto")
|
||||
command.append(build_prompt(max_papers))
|
||||
return command
|
||||
|
||||
|
||||
def build_command(options: RuntimeOptions) -> list[str]:
|
||||
if options.backend == "claude":
|
||||
return build_claude_command()
|
||||
if options.backend == "codex":
|
||||
return build_codex_command(options.max_papers)
|
||||
raise ValueError(f"未対応のAIバックエンドです: {options.backend}")
|
||||
|
||||
|
||||
def require_backend_command(backend: str) -> None:
|
||||
executable = {"claude": "claude", "codex": "codex"}.get(backend)
|
||||
if executable is None:
|
||||
raise ValueError(f"未対応のAIバックエンドです: {backend}")
|
||||
if shutil.which(executable) is None:
|
||||
raise RuntimeError(
|
||||
f"必要なコマンドが見つかりません: {executable}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
options = resolve_options(args)
|
||||
try:
|
||||
CONFIG.validate()
|
||||
require_backend_command(options.backend)
|
||||
command = build_command(options)
|
||||
prompt = build_prompt(options.max_papers)
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=CONFIG.path.project_root,
|
||||
input=(prompt if options.backend == "claude" else None),
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return completed.returncode
|
||||
except Exception as exc:
|
||||
print(f"run_ai.py: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+542
@@ -0,0 +1,542 @@
|
||||
#!/usr/bin/env python3
|
||||
"""tmp/paper-*.jsonをpapers.dbへ一括登録する。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from common import (
|
||||
ALLOWED_CATEGORIES,
|
||||
ALLOWED_PRIORITIES,
|
||||
CONFIG,
|
||||
normalize_doi,
|
||||
normalize_text,
|
||||
now_iso,
|
||||
resolve_pdf_path,
|
||||
sha256_file,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""コマンドライン引数を解析する。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="tmp/paper-*.jsonをpapers.dbへ一括登録する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-dir",
|
||||
default=str(CONFIG.path.tmp_dir),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database",
|
||||
default=str(CONFIG.path.database),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pdf-root",
|
||||
default=str(CONFIG.path.pdf_root),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pattern",
|
||||
default="paper-*.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--continue-on-error",
|
||||
action="store_true",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def backup_database(database: Path) -> Path:
|
||||
"""DBを1回だけバックアップする。"""
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
backup = database.with_name(
|
||||
f"{database.name}.bak-{stamp}"
|
||||
)
|
||||
shutil.copy2(database, backup)
|
||||
return backup
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
"""論文JSONを読み込む。"""
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("JSON root must be an object")
|
||||
return data
|
||||
|
||||
|
||||
def validate_json(data: dict[str, Any]) -> None:
|
||||
"""JSONの必須項目と型を検証する。"""
|
||||
required = ("title", "authors", "year", "journal", "pdf_path")
|
||||
for key in required:
|
||||
if key not in data:
|
||||
raise ValueError(f"missing required field: {key}")
|
||||
|
||||
authors = data["authors"]
|
||||
if not isinstance(authors, list) or not authors:
|
||||
raise ValueError("authors must be a non-empty list")
|
||||
|
||||
relevance = int(data.get("relevance", 0))
|
||||
if relevance not in range(6):
|
||||
raise ValueError("relevance must be 0..5")
|
||||
|
||||
priority = data.get("priority", "D")
|
||||
if priority not in ALLOWED_PRIORITIES:
|
||||
raise ValueError("priority must be A..D")
|
||||
|
||||
category = data.get("category", "Other")
|
||||
if category not in ALLOWED_CATEGORIES:
|
||||
raise ValueError("unknown category")
|
||||
|
||||
|
||||
def add_file_metadata(
|
||||
data: dict[str, Any],
|
||||
pdf_root: Path,
|
||||
) -> None:
|
||||
"""ローカルPDFの情報をJSONへ追加する。"""
|
||||
actual_pdf = resolve_pdf_path(
|
||||
pdf_root,
|
||||
str(data["pdf_path"]),
|
||||
)
|
||||
if not actual_pdf.is_file():
|
||||
raise FileNotFoundError(actual_pdf)
|
||||
|
||||
stat = actual_pdf.stat()
|
||||
data["file_size"] = stat.st_size
|
||||
data["file_mtime"] = datetime.fromtimestamp(
|
||||
stat.st_mtime
|
||||
).astimezone().isoformat(timespec="seconds")
|
||||
data["file_sha256"] = sha256_file(actual_pdf)
|
||||
|
||||
|
||||
def get_category_id(
|
||||
connection: sqlite3.Connection,
|
||||
name: str,
|
||||
) -> int:
|
||||
"""カテゴリIDを取得する。"""
|
||||
connection.execute(
|
||||
"INSERT OR IGNORE INTO categories(name) VALUES (?)",
|
||||
(name,),
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT id FROM categories WHERE name = ?",
|
||||
(name,),
|
||||
).fetchone()
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def get_author_id(
|
||||
connection: sqlite3.Connection,
|
||||
author: dict[str, Any],
|
||||
) -> int:
|
||||
"""著者を取得または登録する。"""
|
||||
name = str(author.get("name", "")).strip()
|
||||
normalized = normalize_text(name)
|
||||
orcid = author.get("orcid")
|
||||
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM authors
|
||||
WHERE name_normalized = ?
|
||||
AND ((orcid = ?) OR (orcid IS NULL AND ? IS NULL))
|
||||
ORDER BY id
|
||||
LIMIT 1
|
||||
""",
|
||||
(normalized, orcid, orcid),
|
||||
).fetchone()
|
||||
|
||||
if row:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE authors
|
||||
SET affiliation = COALESCE(affiliation, ?),
|
||||
orcid = COALESCE(orcid, ?)
|
||||
WHERE id = ?
|
||||
""",
|
||||
(author.get("affiliation"), orcid, row[0]),
|
||||
)
|
||||
return int(row[0])
|
||||
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO authors(
|
||||
name, name_normalized, orcid, affiliation
|
||||
) VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(name, normalized, orcid, author.get("affiliation")),
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
|
||||
def get_keyword_id(
|
||||
connection: sqlite3.Connection,
|
||||
keyword: Any,
|
||||
) -> int:
|
||||
"""キーワードIDを取得する。"""
|
||||
name = " ".join(str(keyword).split())
|
||||
normalized = normalize_text(name)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO keywords(name, name_normalized)
|
||||
VALUES (?, ?)
|
||||
""",
|
||||
(name, normalized),
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT id FROM keywords WHERE name_normalized = ?",
|
||||
(normalized,),
|
||||
).fetchone()
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def find_existing(
|
||||
connection: sqlite3.Connection,
|
||||
data: dict[str, Any],
|
||||
) -> sqlite3.Row | None:
|
||||
"""既存論文をSHA256、DOI、書誌情報の順で探す。"""
|
||||
sha256 = data.get("file_sha256")
|
||||
if sha256:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM papers WHERE file_sha256 = ?",
|
||||
(sha256,),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row
|
||||
|
||||
doi = normalize_doi(data.get("doi"))
|
||||
if doi:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM papers WHERE doi = ?",
|
||||
(doi,),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row
|
||||
|
||||
first_author = (
|
||||
data.get("first_author")
|
||||
or data["authors"][0].get("name", "")
|
||||
)
|
||||
return connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM papers
|
||||
WHERE title_normalized = ?
|
||||
AND COALESCE(first_author, '') = ?
|
||||
AND year IS ?
|
||||
""",
|
||||
(
|
||||
normalize_text(str(data["title"])),
|
||||
first_author,
|
||||
data.get("year"),
|
||||
),
|
||||
).fetchone()
|
||||
|
||||
|
||||
def update_existing_paper(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
category_id: int,
|
||||
data: dict[str, Any],
|
||||
now: str,
|
||||
) -> None:
|
||||
"""既存論文の空欄を補完する。"""
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE papers SET
|
||||
doi = COALESCE(doi, ?),
|
||||
journal = COALESCE(journal, ?),
|
||||
volume = COALESCE(volume, ?),
|
||||
issue = COALESCE(issue, ?),
|
||||
pages = COALESCE(pages, ?),
|
||||
publisher = COALESCE(publisher, ?),
|
||||
article_type = COALESCE(article_type, ?),
|
||||
language = COALESCE(language, ?),
|
||||
abstract = COALESCE(abstract, ?),
|
||||
summary_ja = COALESCE(summary_ja, ?),
|
||||
category_id = COALESCE(category_id, ?),
|
||||
pdf_path = COALESCE(pdf_path, ?),
|
||||
file_size = COALESCE(file_size, ?),
|
||||
file_mtime = COALESCE(file_mtime, ?),
|
||||
file_sha256 = COALESCE(file_sha256, ?),
|
||||
source = COALESCE(source, ?),
|
||||
landing_url = COALESCE(landing_url, ?),
|
||||
pdf_url = COALESCE(pdf_url, ?),
|
||||
url_source = COALESCE(url_source, ?),
|
||||
access_status = COALESCE(access_status, ?),
|
||||
url_checked_at = COALESCE(url_checked_at, ?),
|
||||
checked_at = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
normalize_doi(data.get("doi")),
|
||||
data.get("journal"),
|
||||
data.get("volume"),
|
||||
data.get("issue"),
|
||||
data.get("pages"),
|
||||
data.get("publisher"),
|
||||
data.get("article_type"),
|
||||
data.get("language"),
|
||||
data.get("abstract"),
|
||||
data.get("summary_ja"),
|
||||
category_id,
|
||||
data.get("pdf_path"),
|
||||
data["file_size"],
|
||||
data["file_mtime"],
|
||||
data["file_sha256"],
|
||||
data.get("source"),
|
||||
data.get("landing_url"),
|
||||
data.get("pdf_url"),
|
||||
data.get("url_source"),
|
||||
data.get("access_status"),
|
||||
data.get("url_checked_at"),
|
||||
now,
|
||||
now,
|
||||
paper_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def insert_paper(
|
||||
connection: sqlite3.Connection,
|
||||
category_id: int,
|
||||
data: dict[str, Any],
|
||||
now: str,
|
||||
) -> int:
|
||||
"""新規論文を登録する。"""
|
||||
first_author = (
|
||||
data.get("first_author")
|
||||
or data["authors"][0].get("name", "")
|
||||
)
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO papers(
|
||||
doi, title, title_normalized, first_author, year, journal,
|
||||
volume, issue, pages, publisher, article_type, language,
|
||||
abstract, summary_ja, relevance, priority, review_status,
|
||||
category_id, pdf_path, file_size, file_mtime, file_sha256,
|
||||
source, landing_url, pdf_url, url_source, access_status,
|
||||
url_checked_at, created_at, updated_at, checked_at, notes
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
""",
|
||||
(
|
||||
normalize_doi(data.get("doi")),
|
||||
data["title"],
|
||||
normalize_text(str(data["title"])),
|
||||
first_author,
|
||||
data.get("year"),
|
||||
data.get("journal"),
|
||||
data.get("volume"),
|
||||
data.get("issue"),
|
||||
data.get("pages"),
|
||||
data.get("publisher"),
|
||||
data.get("article_type"),
|
||||
data.get("language"),
|
||||
data.get("abstract"),
|
||||
data.get("summary_ja"),
|
||||
int(data.get("relevance", 0)),
|
||||
data.get("priority", "D"),
|
||||
data.get("review_status", "unreviewed"),
|
||||
category_id,
|
||||
data.get("pdf_path"),
|
||||
data["file_size"],
|
||||
data["file_mtime"],
|
||||
data["file_sha256"],
|
||||
data.get("source"),
|
||||
data.get("landing_url"),
|
||||
data.get("pdf_url"),
|
||||
data.get("url_source"),
|
||||
data.get("access_status"),
|
||||
data.get("url_checked_at"),
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
data.get("notes"),
|
||||
),
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
|
||||
def register_authors(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
authors: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""論文著者を登録する。"""
|
||||
existing = connection.execute(
|
||||
"SELECT 1 FROM paper_authors WHERE paper_id = ? LIMIT 1",
|
||||
(paper_id,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
return
|
||||
|
||||
for order, author in enumerate(authors, start=1):
|
||||
author_id = get_author_id(connection, author)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO paper_authors(
|
||||
paper_id, author_id, author_order, is_corresponding
|
||||
) VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
paper_id,
|
||||
author_id,
|
||||
order,
|
||||
int(bool(author.get("is_corresponding", False))),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def register_keywords(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
keywords: list[Any],
|
||||
) -> None:
|
||||
"""論文キーワードを登録する。"""
|
||||
for keyword in keywords:
|
||||
keyword_id = get_keyword_id(connection, keyword)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO paper_keywords(
|
||||
paper_id, keyword_id
|
||||
) VALUES (?, ?)
|
||||
""",
|
||||
(paper_id, keyword_id),
|
||||
)
|
||||
|
||||
|
||||
def register_one(
|
||||
connection: sqlite3.Connection,
|
||||
json_path: Path,
|
||||
pdf_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""1件のJSONを登録する。"""
|
||||
data = load_json(json_path)
|
||||
validate_json(data)
|
||||
add_file_metadata(data, pdf_root)
|
||||
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
existing = find_existing(connection, data)
|
||||
category_id = get_category_id(
|
||||
connection,
|
||||
str(data.get("category", "Other")),
|
||||
)
|
||||
now = now_iso()
|
||||
|
||||
if existing:
|
||||
paper_id = int(existing["id"])
|
||||
update_existing_paper(
|
||||
connection,
|
||||
paper_id,
|
||||
category_id,
|
||||
data,
|
||||
now,
|
||||
)
|
||||
status = "updated"
|
||||
else:
|
||||
paper_id = insert_paper(
|
||||
connection,
|
||||
category_id,
|
||||
data,
|
||||
now,
|
||||
)
|
||||
status = "registered"
|
||||
|
||||
register_authors(connection, paper_id, data["authors"])
|
||||
register_keywords(
|
||||
connection,
|
||||
paper_id,
|
||||
data.get("keywords", []),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO processing_log(
|
||||
pdf_path, paper_id, status, message, processed_at
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(data["pdf_path"], paper_id, status, status, now),
|
||||
)
|
||||
|
||||
integrity = connection.execute(
|
||||
"PRAGMA integrity_check"
|
||||
).fetchone()[0]
|
||||
if integrity != "ok":
|
||||
raise RuntimeError(integrity)
|
||||
|
||||
connection.commit()
|
||||
return {
|
||||
"status": status,
|
||||
"paper_id": paper_id,
|
||||
"integrity_check": integrity,
|
||||
}
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def process_files(args: argparse.Namespace) -> int:
|
||||
"""対象JSONを一括処理する。"""
|
||||
input_dir = Path(args.input_dir)
|
||||
database = Path(args.database)
|
||||
pdf_root = Path(args.pdf_root).expanduser().resolve()
|
||||
files = sorted(input_dir.glob(args.pattern))
|
||||
|
||||
if not files:
|
||||
print("新規JSONはありません。")
|
||||
return 0
|
||||
if not database.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"database not found: {database}"
|
||||
)
|
||||
|
||||
if CONFIG.processing.backup_database:
|
||||
backup = backup_database(database)
|
||||
print(f"Backup: {backup}")
|
||||
|
||||
counts = {"registered": 0, "updated": 0, "failed": 0}
|
||||
with sqlite3.connect(database) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
|
||||
for path in files:
|
||||
print(f"=== {path} ===")
|
||||
try:
|
||||
result = register_one(connection, path, pdf_root)
|
||||
counts[result["status"]] += 1
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
except Exception as exc:
|
||||
counts["failed"] += 1
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
if not args.continue_on_error:
|
||||
return 1
|
||||
|
||||
print()
|
||||
print("=== 一括登録結果 ===")
|
||||
print(f"対象: {len(files)}")
|
||||
print(f"新規登録: {counts['registered']}")
|
||||
print(f"既存更新: {counts['updated']}")
|
||||
print(f"失敗: {counts['failed']}")
|
||||
return 1 if counts["failed"] else 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""メイン処理。"""
|
||||
try:
|
||||
return process_files(parse_args())
|
||||
except Exception as exc:
|
||||
print(f"update_all.py: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+442
@@ -0,0 +1,442 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SHA256が同じPDFの移動・改名をpapers.dbへ反映する。
|
||||
|
||||
PDF内容が変わってSHA256も変わった場合は更新しない。
|
||||
その場合はdelete_paper.pyでDBレコードを削除し、
|
||||
通常の登録フローで再登録する。
|
||||
|
||||
既定はdry-run。実際に更新する場合は--applyを指定する。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from common import CONFIG, resolve_pdf_path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfFile:
|
||||
relative_path: str
|
||||
sha256: str
|
||||
file_size: int
|
||||
file_mtime: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PathUpdate:
|
||||
paper_id: int
|
||||
title: str
|
||||
old_path: str
|
||||
new_path: str
|
||||
sha256: str
|
||||
file_size: int
|
||||
file_mtime: float
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="SHA256が同じPDFの移動・改名をDBへ反映する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database",
|
||||
type=Path,
|
||||
default=CONFIG.path.database,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pdf-root",
|
||||
type=Path,
|
||||
default=CONFIG.path.pdf_root,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scan-all",
|
||||
action="store_true",
|
||||
help="PDF_ROOT全体を走査する",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="更新を実際にDBへ反映する",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-backup",
|
||||
action="store_true",
|
||||
help="更新前バックアップを作成しない",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_paths(database: Path, pdf_root: Path) -> None:
|
||||
if not database.is_file():
|
||||
raise FileNotFoundError(database)
|
||||
|
||||
if not pdf_root.is_dir():
|
||||
raise NotADirectoryError(pdf_root)
|
||||
|
||||
|
||||
def scan_roots(
|
||||
pdf_root: Path,
|
||||
scan_all: bool,
|
||||
) -> tuple[Path, ...]:
|
||||
if scan_all:
|
||||
return (pdf_root,)
|
||||
|
||||
roots = tuple(
|
||||
pdf_root / name
|
||||
for name in CONFIG.processing.scan_directories
|
||||
)
|
||||
|
||||
for root in roots:
|
||||
if not root.is_dir():
|
||||
raise NotADirectoryError(root)
|
||||
|
||||
return roots
|
||||
|
||||
|
||||
def iter_pdf_paths(
|
||||
roots: Iterable[Path],
|
||||
) -> Iterable[Path]:
|
||||
seen: set[Path] = set()
|
||||
|
||||
for root in roots:
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
|
||||
if path.suffix.lower() != ".pdf":
|
||||
continue
|
||||
|
||||
resolved = path.resolve()
|
||||
|
||||
if resolved in seen:
|
||||
continue
|
||||
|
||||
seen.add(resolved)
|
||||
yield resolved
|
||||
|
||||
|
||||
def calculate_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
|
||||
with path.open("rb") as file:
|
||||
for chunk in iter(
|
||||
lambda: file.read(1024 * 1024),
|
||||
b"",
|
||||
):
|
||||
digest.update(chunk)
|
||||
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def inspect_pdf(
|
||||
path: Path,
|
||||
pdf_root: Path,
|
||||
) -> PdfFile:
|
||||
stat = path.stat()
|
||||
|
||||
return PdfFile(
|
||||
relative_path=path.relative_to(
|
||||
pdf_root
|
||||
).as_posix(),
|
||||
sha256=calculate_sha256(path),
|
||||
file_size=stat.st_size,
|
||||
file_mtime=stat.st_mtime,
|
||||
)
|
||||
|
||||
|
||||
def build_sha_index(
|
||||
pdf_root: Path,
|
||||
roots: tuple[Path, ...],
|
||||
) -> dict[str, list[PdfFile]]:
|
||||
index: dict[str, list[PdfFile]] = defaultdict(list)
|
||||
|
||||
paths = sorted(
|
||||
iter_pdf_paths(roots),
|
||||
key=lambda item: str(item).casefold(),
|
||||
)
|
||||
|
||||
for number, path in enumerate(paths, start=1):
|
||||
pdf = inspect_pdf(path, pdf_root)
|
||||
index[pdf.sha256].append(pdf)
|
||||
|
||||
print(
|
||||
f"\rPDFを走査中: {number}件",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if paths:
|
||||
print()
|
||||
|
||||
return dict(index)
|
||||
|
||||
|
||||
def open_database(
|
||||
database: Path,
|
||||
) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
|
||||
def load_papers(
|
||||
connection: sqlite3.Connection,
|
||||
) -> list[sqlite3.Row]:
|
||||
sql = """
|
||||
SELECT id, title, pdf_path, file_sha256
|
||||
FROM papers
|
||||
WHERE file_sha256 IS NOT NULL
|
||||
AND TRIM(file_sha256) <> ''
|
||||
ORDER BY id
|
||||
"""
|
||||
return connection.execute(sql).fetchall()
|
||||
|
||||
|
||||
def current_path_is_valid(
|
||||
pdf_root: Path,
|
||||
row: sqlite3.Row,
|
||||
) -> bool:
|
||||
if not row["pdf_path"]:
|
||||
return False
|
||||
|
||||
try:
|
||||
path = resolve_pdf_path(
|
||||
pdf_root,
|
||||
row["pdf_path"],
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if not path.is_file():
|
||||
return False
|
||||
|
||||
return (
|
||||
calculate_sha256(path)
|
||||
== row["file_sha256"].strip().lower()
|
||||
)
|
||||
|
||||
|
||||
def find_updates(
|
||||
rows: Iterable[sqlite3.Row],
|
||||
index: dict[str, list[PdfFile]],
|
||||
pdf_root: Path,
|
||||
) -> tuple[
|
||||
list[PathUpdate],
|
||||
list[sqlite3.Row],
|
||||
list[sqlite3.Row],
|
||||
]:
|
||||
updates: list[PathUpdate] = []
|
||||
ambiguous: list[sqlite3.Row] = []
|
||||
not_found: list[sqlite3.Row] = []
|
||||
|
||||
for row in rows:
|
||||
if current_path_is_valid(pdf_root, row):
|
||||
continue
|
||||
|
||||
sha256 = row["file_sha256"].strip().lower()
|
||||
candidates = index.get(sha256, [])
|
||||
|
||||
if not candidates:
|
||||
not_found.append(row)
|
||||
continue
|
||||
|
||||
if len(candidates) != 1:
|
||||
ambiguous.append(row)
|
||||
continue
|
||||
|
||||
candidate = candidates[0]
|
||||
|
||||
updates.append(
|
||||
PathUpdate(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
old_path=row["pdf_path"] or "",
|
||||
new_path=candidate.relative_path,
|
||||
sha256=sha256,
|
||||
file_size=candidate.file_size,
|
||||
file_mtime=candidate.file_mtime,
|
||||
)
|
||||
)
|
||||
|
||||
return updates, ambiguous, not_found
|
||||
|
||||
|
||||
def print_updates(
|
||||
updates: Iterable[PathUpdate],
|
||||
) -> None:
|
||||
items = list(updates)
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
print()
|
||||
print("=== 更新候補 ===")
|
||||
|
||||
for item in items:
|
||||
print(f"id: {item.paper_id}")
|
||||
print(f"title: {item.title}")
|
||||
print(f"old: {item.old_path or '(empty)'}")
|
||||
print(f"new: {item.new_path}")
|
||||
print()
|
||||
|
||||
|
||||
def print_rows(
|
||||
title: str,
|
||||
rows: Iterable[sqlite3.Row],
|
||||
) -> None:
|
||||
items = list(rows)
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
print()
|
||||
print(f"=== {title} ===")
|
||||
|
||||
for row in items:
|
||||
print(
|
||||
f"id={row['id']}: "
|
||||
f"{row['pdf_path']} "
|
||||
f"({row['title']})"
|
||||
)
|
||||
|
||||
|
||||
def create_backup(database: Path) -> Path:
|
||||
backup = database.with_name(
|
||||
database.name + ".before-path-update"
|
||||
)
|
||||
|
||||
counter = 1
|
||||
|
||||
while backup.exists():
|
||||
backup = database.with_name(
|
||||
database.name
|
||||
+ f".before-path-update.{counter}"
|
||||
)
|
||||
counter += 1
|
||||
|
||||
shutil.copy2(database, backup)
|
||||
return backup
|
||||
|
||||
|
||||
def apply_updates(
|
||||
connection: sqlite3.Connection,
|
||||
updates: Iterable[PathUpdate],
|
||||
) -> int:
|
||||
sql = """
|
||||
UPDATE papers
|
||||
SET pdf_path = ?,
|
||||
file_size = ?,
|
||||
file_mtime = ?
|
||||
WHERE id = ?
|
||||
AND lower(file_sha256) = ?
|
||||
"""
|
||||
|
||||
count = 0
|
||||
|
||||
with connection:
|
||||
for item in updates:
|
||||
cursor = connection.execute(
|
||||
sql,
|
||||
(
|
||||
item.new_path,
|
||||
item.file_size,
|
||||
item.file_mtime,
|
||||
item.paper_id,
|
||||
item.sha256,
|
||||
),
|
||||
)
|
||||
count += cursor.rowcount
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
try:
|
||||
database = args.database.expanduser().resolve()
|
||||
pdf_root = args.pdf_root.expanduser().resolve()
|
||||
|
||||
validate_paths(database, pdf_root)
|
||||
|
||||
index = build_sha_index(
|
||||
pdf_root,
|
||||
scan_roots(pdf_root, args.scan_all),
|
||||
)
|
||||
|
||||
with open_database(database) as connection:
|
||||
rows = load_papers(connection)
|
||||
updates, ambiguous, not_found = (
|
||||
find_updates(
|
||||
rows,
|
||||
index,
|
||||
pdf_root,
|
||||
)
|
||||
)
|
||||
|
||||
print_updates(updates)
|
||||
print_rows(
|
||||
"同一SHA256の候補が複数",
|
||||
ambiguous,
|
||||
)
|
||||
print_rows(
|
||||
"SHA256一致ファイルなし",
|
||||
not_found,
|
||||
)
|
||||
|
||||
if args.apply and updates:
|
||||
if not args.no_backup:
|
||||
backup = create_backup(database)
|
||||
print(f"DBバックアップ: {backup}")
|
||||
|
||||
changed = apply_updates(
|
||||
connection,
|
||||
updates,
|
||||
)
|
||||
print(f"DB更新: {changed} 件")
|
||||
|
||||
print()
|
||||
print("=== 概要 ===")
|
||||
print(
|
||||
"走査PDF: "
|
||||
f"{sum(len(v) for v in index.values())} 件"
|
||||
)
|
||||
print(f"DB照合対象: {len(rows)} 件")
|
||||
print(f"更新候補: {len(updates)} 件")
|
||||
print(f"曖昧な一致: {len(ambiguous)} 件")
|
||||
print(f"一致なし: {len(not_found)} 件")
|
||||
print(
|
||||
"実行モード: "
|
||||
+ ("更新実行" if args.apply else "dry-run")
|
||||
)
|
||||
|
||||
if updates and not args.apply:
|
||||
print()
|
||||
print(
|
||||
"更新するには --apply を指定してください。"
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"update_pdf_paths.py: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+312
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from common import CONFIG, resolve_pdf_path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MissingPdf:
|
||||
"""見つからないPDFの情報。"""
|
||||
|
||||
paper_id: int
|
||||
title: str
|
||||
pdf_path: str
|
||||
resolved_path: str | None
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IntegrityResult:
|
||||
"""SQLite整合性検証の結果。"""
|
||||
|
||||
integrity: str
|
||||
foreign_key_count: int
|
||||
errors: tuple[str, ...]
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""コマンドライン引数を解析する。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="papers.dbの整合性を検証する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"database",
|
||||
nargs="?",
|
||||
default=str(CONFIG.path.database),
|
||||
help="検証するSQLiteデータベース",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pdf-root",
|
||||
default=str(CONFIG.path.pdf_root),
|
||||
help="PDF_ROOTのパス",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def open_database(database: Path) -> sqlite3.Connection:
|
||||
"""SQLiteデータベースへ接続する。"""
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
|
||||
def check_integrity(
|
||||
connection: sqlite3.Connection,
|
||||
) -> IntegrityResult:
|
||||
"""SQLite本体と外部キーの整合性を検証する。"""
|
||||
errors: list[str] = []
|
||||
|
||||
integrity = connection.execute(
|
||||
"PRAGMA integrity_check"
|
||||
).fetchone()[0]
|
||||
|
||||
if integrity != "ok":
|
||||
errors.append(f"integrity_check: {integrity}")
|
||||
|
||||
foreign_keys = connection.execute(
|
||||
"PRAGMA foreign_key_check"
|
||||
).fetchall()
|
||||
|
||||
foreign_key_count = len(foreign_keys)
|
||||
|
||||
if foreign_key_count:
|
||||
errors.append(
|
||||
f"foreign keys: {foreign_key_count}"
|
||||
)
|
||||
|
||||
return IntegrityResult(
|
||||
integrity=integrity,
|
||||
foreign_key_count=foreign_key_count,
|
||||
errors=tuple(errors),
|
||||
)
|
||||
|
||||
|
||||
def validation_queries() -> tuple[tuple[str, str], ...]:
|
||||
"""DB内容の検証用SQLを返す。"""
|
||||
return (
|
||||
(
|
||||
"duplicate DOI",
|
||||
"""
|
||||
SELECT doi
|
||||
FROM papers
|
||||
WHERE doi IS NOT NULL
|
||||
AND doi <> ''
|
||||
GROUP BY doi
|
||||
HAVING COUNT(*) > 1
|
||||
""",
|
||||
),
|
||||
(
|
||||
"duplicate SHA256",
|
||||
"""
|
||||
SELECT file_sha256
|
||||
FROM papers
|
||||
WHERE file_sha256 IS NOT NULL
|
||||
AND file_sha256 <> ''
|
||||
GROUP BY file_sha256
|
||||
HAVING COUNT(*) > 1
|
||||
""",
|
||||
),
|
||||
(
|
||||
"empty title",
|
||||
"""
|
||||
SELECT id
|
||||
FROM papers
|
||||
WHERE title IS NULL
|
||||
OR TRIM(title) = ''
|
||||
""",
|
||||
),
|
||||
(
|
||||
"invalid relevance",
|
||||
"""
|
||||
SELECT id
|
||||
FROM papers
|
||||
WHERE relevance NOT BETWEEN 0 AND 5
|
||||
""",
|
||||
),
|
||||
(
|
||||
"invalid priority",
|
||||
"""
|
||||
SELECT id
|
||||
FROM papers
|
||||
WHERE priority NOT IN ('A', 'B', 'C', 'D')
|
||||
""",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def check_database_values(
|
||||
connection: sqlite3.Connection,
|
||||
) -> list[str]:
|
||||
"""重複や値域を検証する。"""
|
||||
errors: list[str] = []
|
||||
|
||||
for label, sql in validation_queries():
|
||||
rows = connection.execute(sql).fetchall()
|
||||
|
||||
if rows:
|
||||
errors.append(f"{label}: {len(rows)}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def iter_papers_with_pdf(
|
||||
connection: sqlite3.Connection,
|
||||
) -> Iterable[sqlite3.Row]:
|
||||
"""PDFパスが登録された論文を返す。"""
|
||||
sql = """
|
||||
SELECT id, title, pdf_path
|
||||
FROM papers
|
||||
WHERE pdf_path IS NOT NULL
|
||||
AND TRIM(pdf_path) <> ''
|
||||
ORDER BY id
|
||||
"""
|
||||
return connection.execute(sql)
|
||||
|
||||
|
||||
def inspect_pdf(
|
||||
pdf_root: Path,
|
||||
row: sqlite3.Row,
|
||||
) -> MissingPdf | None:
|
||||
"""1件のPDFが存在するか確認する。"""
|
||||
try:
|
||||
path = resolve_pdf_path(
|
||||
pdf_root,
|
||||
row["pdf_path"],
|
||||
)
|
||||
|
||||
if path.is_file():
|
||||
return None
|
||||
|
||||
return MissingPdf(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
pdf_path=row["pdf_path"],
|
||||
resolved_path=str(path),
|
||||
reason="file does not exist",
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
return MissingPdf(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
pdf_path=row["pdf_path"],
|
||||
resolved_path=None,
|
||||
reason=f"{type(exc).__name__}: {exc}",
|
||||
)
|
||||
|
||||
|
||||
def find_missing_pdfs(
|
||||
connection: sqlite3.Connection,
|
||||
pdf_root: Path,
|
||||
) -> list[MissingPdf]:
|
||||
"""存在しないPDFを検索する。"""
|
||||
missing: list[MissingPdf] = []
|
||||
|
||||
for row in iter_papers_with_pdf(connection):
|
||||
item = inspect_pdf(pdf_root, row)
|
||||
|
||||
if item is not None:
|
||||
missing.append(item)
|
||||
|
||||
return missing
|
||||
|
||||
|
||||
def print_missing_pdfs(
|
||||
missing: Iterable[MissingPdf],
|
||||
) -> None:
|
||||
"""見つからないPDFの詳細を表示する。"""
|
||||
items = list(missing)
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
print()
|
||||
print("=== Missing PDFs ===")
|
||||
|
||||
for item in items:
|
||||
print(f"paper_id: {item.paper_id}")
|
||||
print(f"title: {item.title}")
|
||||
print(f"pdf_path: {item.pdf_path}")
|
||||
print(
|
||||
"resolved_path:",
|
||||
item.resolved_path
|
||||
or "(resolve failed)",
|
||||
)
|
||||
print(f"reason: {item.reason}")
|
||||
print()
|
||||
|
||||
|
||||
def print_summary(
|
||||
integrity: IntegrityResult,
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
"""検証結果を表示する。"""
|
||||
print(f"integrity_check: {integrity.integrity}")
|
||||
print(
|
||||
"foreign_key_violations:",
|
||||
integrity.foreign_key_count,
|
||||
)
|
||||
print(f"errors: {len(errors)}")
|
||||
|
||||
for error in errors:
|
||||
print(f"- {error}")
|
||||
|
||||
|
||||
def validate(
|
||||
database: Path,
|
||||
pdf_root: Path,
|
||||
) -> list[str]:
|
||||
"""データベース全体を検証する。"""
|
||||
with open_database(database) as connection:
|
||||
integrity = check_integrity(connection)
|
||||
value_errors = check_database_values(
|
||||
connection
|
||||
)
|
||||
missing = find_missing_pdfs(
|
||||
connection,
|
||||
pdf_root,
|
||||
)
|
||||
|
||||
errors = list(integrity.errors)
|
||||
errors.extend(value_errors)
|
||||
|
||||
if missing:
|
||||
errors.append(
|
||||
f"missing PDF: {len(missing)}"
|
||||
)
|
||||
|
||||
print_missing_pdfs(missing)
|
||||
print_summary(integrity, errors)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""メイン処理。"""
|
||||
args = parse_args()
|
||||
|
||||
database = Path(
|
||||
args.database
|
||||
).expanduser().resolve()
|
||||
|
||||
pdf_root = Path(
|
||||
args.pdf_root
|
||||
).expanduser().resolve()
|
||||
|
||||
errors = validate(
|
||||
database,
|
||||
pdf_root,
|
||||
)
|
||||
|
||||
return 1 if errors else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user