From 9e3b3ddc2976e1eb145ce89dd37b902168886c4f Mon Sep 17 00:00:00 2001 From: Takahiro OHKUBO Date: Thu, 30 Jul 2026 10:51:43 +0900 Subject: [PATCH] Initial commit --- .gitignore | 41 + LICENSE | 21 + README.org | 46 + examples/wglass_db/README.org | 40 + examples/wglass_db/config.py | 113 ++ paper_db_common/CLAUDE.md | 237 ++++ paper_db_common/README.org | 75 ++ paper_db_common/new_project.py | 80 ++ paper_db_common/paperdb_config.py | 159 +++ paper_db_common/project_loader.py | 47 + paper_db_common/publish.py | 323 +++++ paper_db_common/scripts/common.py | 71 ++ paper_db_common/scripts/delete_paper.py | 269 +++++ paper_db_common/scripts/export_org.py | 205 ++++ paper_db_common/scripts/extract_text.py | 1137 ++++++++++++++++++ paper_db_common/scripts/init_db.py | 161 +++ paper_db_common/scripts/read_config.py | 36 + paper_db_common/scripts/run_ai.py | 220 ++++ paper_db_common/scripts/update_all.py | 542 +++++++++ paper_db_common/scripts/update_pdf_paths.py | 442 +++++++ paper_db_common/scripts/validate_db.py | 312 +++++ paper_db_common/templates/project/README.org | 40 + paper_db_common/templates/project/config.py | 88 ++ 23 files changed, 4705 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.org create mode 100644 examples/wglass_db/README.org create mode 100644 examples/wglass_db/config.py create mode 100644 paper_db_common/CLAUDE.md create mode 100644 paper_db_common/README.org create mode 100755 paper_db_common/new_project.py create mode 100644 paper_db_common/paperdb_config.py create mode 100644 paper_db_common/project_loader.py create mode 100755 paper_db_common/publish.py create mode 100755 paper_db_common/scripts/common.py create mode 100755 paper_db_common/scripts/delete_paper.py create mode 100755 paper_db_common/scripts/export_org.py create mode 100755 paper_db_common/scripts/extract_text.py create mode 100755 paper_db_common/scripts/init_db.py create mode 100755 paper_db_common/scripts/read_config.py create mode 100755 paper_db_common/scripts/run_ai.py create mode 100755 paper_db_common/scripts/update_all.py create mode 100755 paper_db_common/scripts/update_pdf_paths.py create mode 100755 paper_db_common/scripts/validate_db.py create mode 100644 paper_db_common/templates/project/README.org create mode 100755 paper_db_common/templates/project/config.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e8d6799 --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# Python +__pycache__/ +*.py[cod] +*.so +*.egg-info/ +.venv/ +venv/ + +# macOS +.DS_Store + +# Editors +*~ +\#*\# +.\#* +.vscode/ +.idea/ + +# Runtime +logs/ +tmp/ + +# Databases +*.db +*.db-journal +*.sqlite +*.sqlite3 + +# Generated +papers.org +summary.org +paper-*.json + +# Cache +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# Build +build/ +dist/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..69f8c62 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Takahiro Ohkubo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.org b/README.org new file mode 100644 index 0000000..1078d75 --- /dev/null +++ b/README.org @@ -0,0 +1,46 @@ +#+TITLE: paper_db + +* paper_db + +paper_db は、複数分野で共通利用できる論文データベース基盤です。 + +** 特徴 + +- PDFから本文抽出 +- Claude / Codex による書誌情報・要約生成 +- SQLiteで論文管理 +- Org-modeへのエクスポート +- 分野ごとの差分は config.py のみ + +** ディレクトリ構成 + +#+begin_example +paper_db/ +├── paper_db_common/ +├── examples/ +│ └── wglass_db/ +└── README.org +#+end_example + +** 新しいプロジェクトの作成 + +#+begin_src sh +paper_db_common/new_project.py ~/research/my_db +#+end_src + +** 論文処理 + +#+begin_src sh +cd ~/research/my_db +/path/to/paper_db_common/publish.py process +#+end_src + +** 公開 + +#+begin_src sh +/path/to/paper_db_common/publish.py publish +#+end_src + +** ライセンス + +MIT License diff --git a/examples/wglass_db/README.org b/examples/wglass_db/README.org new file mode 100644 index 0000000..cf8c90c --- /dev/null +++ b/examples/wglass_db/README.org @@ -0,0 +1,40 @@ +#+TITLE: 論文データベースプロジェクト + +* 構成 + +このディレクトリには、分野固有の設定とデータだけを置く。 + +#+begin_example +config.py +papers.db +papers.org +papers.publish.db +logs/ +tmp/ +#+end_example + +=CLAUDE.md= とPythonスクリプトは共通側に置かれるため、 +このディレクトリには作成しない。 + +* 初期設定 + +=config.py= の次の項目を編集する。 + +- =pdf_root= +- =scan_directories= +- =domain= +- 必要な場合だけ =remote= + +* 実行 + +プロジェクトディレクトリへ移動して実行する。 + +#+begin_src sh +/path/to/paper_db_common/publish.py +#+end_src + +公開用DBとPDFも転送する場合は次を使う。 + +#+begin_src sh +/path/to/paper_db_common/publish.py --publish +#+end_src diff --git a/examples/wglass_db/config.py b/examples/wglass_db/config.py new file mode 100644 index 0000000..451508d --- /dev/null +++ b/examples/wglass_db/config.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +from pathlib import Path + +from paperdb_config import AIConfig +from paperdb_config import Config +from paperdb_config import DomainConfig +from paperdb_config import PathConfig +from paperdb_config import ProcessingConfig +from paperdb_config import RemoteConfig + + +ROOT = Path(__file__).resolve().parent + +CONFIG = Config( + path=PathConfig( + project_root=ROOT, + pdf_root=( + Path.home() + / "G_drive" + / "ガラス固化体データベース" + ), + database=ROOT / "papers.db", + publish_database=ROOT / "papers.publish.db", + tmp_dir=ROOT / "tmp", + text_dir=ROOT / "tmp" / "text", + log_dir=ROOT / "logs", + papers_org=ROOT / "papers.org", + ), + remote=RemoteConfig( + host="wglass@amorphous", + database_dir="/home/wglass/public_html/db", + pdf_dir="/home/wglass/public_html/pdf", + ), + ai=AIConfig( + backend="claude", + claude_permission_mode="dontAsk", + claude_allowed_tools=( + "Read", + "Glob", + "Grep", + "Write", + "Edit", + "WebSearch", + "WebFetch", + ), + codex_model=None, + codex_sandbox="workspace-write", + codex_full_auto=True, + ), + processing=ProcessingConfig( + scan_directories=( + "千葉大_大窪", + "JAEA_三ツ井", + "RWMC_稲垣", + ), + ignore_directories=("old", "tmp", "trash"), + max_papers=1, + initial_pages=2, + fallback_pages=5, + min_text_length=1500, + update_database=True, + validate_database=True, + export_org=True, + backup_database=True, + ), + domain=DomainConfig( + name="ガラス固化体", + description=( + "高レベル放射性廃棄物ガラス、廃棄物ガラス、" + "ホウケイ酸塩ガラス、耐久性、変質、放射線影響、" + "核種保持に関する論文" + ), + categories=( + "Review", + "Glass structure", + "Crystallization", + "Melting", + "Corrosion", + "Durability", + "Alteration layer", + "Radiation effects", + "Simulation", + "Molecular dynamics", + "Machine learning", + "NMR", + "Raman", + "XAFS", + "Diffraction", + "Actinides", + "Fission products", + "Other", + ), + default_category="Other", + summary_length_ja="100〜200字程度", + summary_focus=( + "研究目的", + "使用した実験または計算手法", + "主な結果", + "新規性", + "ガラス固化体研究との関連", + ), + relevance_criteria=( + "無関係", + "わずかに関連", + "周辺分野", + "関連するガラス科学", + "ガラス固化体研究へ直接応用可能", + "高レベル放射性廃棄物ガラスそのもの", + ), + ), +) diff --git a/paper_db_common/CLAUDE.md b/paper_db_common/CLAUDE.md new file mode 100644 index 0000000..90cc8eb --- /dev/null +++ b/paper_db_common/CLAUDE.md @@ -0,0 +1,237 @@ +# 論文データベース共通処理 + +## 1. 役割 + +このプロジェクトはPDFファイルそのものを保持しない。 + +PDFは`config.py`の`CONFIG.path.pdf_root`で指定された外部ディレクトリに +保存されている。 + +`papers.db`を唯一のマスターデータベースとする。 +`papers.org`は閲覧・検索・レビュー用であり、マスターデータではない。 + +分野名、カテゴリ、要約観点、relevance基準は`config.py`の +`CONFIG.domain`を唯一の定義元とする。 + +## 2. 処理前に必ず行うこと + +1. プロジェクトの`config.py`を読む。 +2. `CONFIG`を読み、処理条件と分野設定を確認する。 +3. 処理対象は`publish.py`が`tmp/text/`へ準備する。 +4. `tmp/text/`にない論文は処理しない。 +5. 設定不足や存在しないパスがあれば処理を開始しない。 + +## 3. TASK=process_new_papers + +AIが行う処理は、`tmp/text/`の解析とJSON生成までとする。 + +1. `tmp/text/paper-*.txt`だけを処理対象とする。 +2. 各テキスト先頭の`PDF_PATH`をJSONの`pdf_path`に使用する。 +3. 書誌情報、要約、キーワード、分類、Webリンクを抽出する。 +4. 各論文について`tmp/paper-*.json`を生成する。 +5. 作成したJSONファイルと処理結果を報告する。 + +原則として元PDFは直接読まない。次の場合のみ、`PDF_PATH`に +記載された1ファイルを参照してよい。 + +- `PDF_TEXT_EMPTY`が`yes` +- `TEXT_INSUFFICIENT`が`yes` +- テキストが文字化けしている +- タイトル、著者、発行年、要約を確認できない + +他のPDFやディレクトリは探索しない。 + +AIは次を実行しない。 + +- DB登録スクリプト +- DB検証スクリプト +- Org出力スクリプト +- `papers.db`の直接更新 + +JSON生成後のDB登録、検証、Org出力は`publish.py`が実行する。 + +## 4. PDFパス + +`papers.db`の`pdf_path`には、`CONFIG.path.pdf_root`からの +相対パスだけを保存する。 + +絶対パスは保存しない。PDFは削除、移動、改名しない。 + +## 5. 抽出項目 + +必須: + +- title +- authors +- first_author +- year +- journal +- doi +- pdf_path + +可能なら: + +- volume +- issue +- pages +- publisher +- article_type +- language +- abstract +- ORCID +- affiliation + +AIが付与: + +- summary_ja +- keywords +- category +- relevance +- priority + +## 6. JSONフォーマット + +```json +{ + "doi": null, + "title": "", + "authors": [ + { + "name": "", + "orcid": null, + "affiliation": null, + "is_corresponding": false + } + ], + "first_author": "", + "year": null, + "journal": "", + "volume": null, + "issue": null, + "pages": null, + "publisher": null, + "article_type": "journal article", + "language": "en", + "abstract": null, + "summary_ja": "", + "keywords": [], + "category": "Other", + "relevance": 3, + "priority": "C", + "pdf_path": "", + "source": "local PDF", + "landing_url": null, + "pdf_url": null, + "url_source": "unknown", + "access_status": "unknown", + "url_checked_at": null +} +``` + +### authors + +`authors`は必ず配列とする。複数著者は論文中の順番で並べる。 +取得できない場合は、`name`を`unknown`とした要素を1件入れる。 + +### first_author + +`authors`配列の先頭の`name`と一致させる。 +著者不明の場合は`unknown`とする。 + +### keywords + +必ず文字列の配列とする。取得できない場合は空配列とする。 + +### category + +`config.py`の`CONFIG.domain.categories`から、最も適切なものを +1つ選択する。該当するカテゴリがない場合は +`CONFIG.domain.default_category`を使用する。 + +### DOI + +- 推測しない。 +- 小文字化する。 +- `https://doi.org/`などの接頭辞を除去する。 +- 不明な場合は`null`とする。 + +### year + +整数または`null`とする。 + +### article_type + +例: + +- journal article +- review +- report +- conference paper +- book chapter +- other + +## 7. Web上の論文リンク + +DOIがある場合、`landing_url`は`https://doi.org/{doi}`とする。 +PDF URLは推測しない。実際に公開されていることを確認できた場合のみ +`pdf_url`へ保存する。 + +`url_source`の例: + +- doi +- crossref +- unpaywall +- publisher +- institutional_repository +- government_repository +- manual +- unknown + +`access_status`: + +- open +- restricted +- unknown + +## 8. 日本語要約 + +`CONFIG.domain.summary_length_ja`を目安とする。 +`CONFIG.domain.summary_focus`に記載された観点を優先する。 +本文から確認できないことは推測しない。 + +## 9. RelevanceとPriority + +`relevance`は0から5の整数とする。 +判断基準は`CONFIG.domain.relevance_criteria`を使用する。 + +Priorityは次の対応に固定する。 + +- A: relevance=5 +- B: relevance=4 +- C: relevance=3 +- D: relevance=0から2 + +## 10. JSONの妥当性 + +次を厳守する。 + +- `authors`: 空でない配列 +- `keywords`: 配列 +- `year`: 整数または`null` +- `doi`: 文字列または`null` +- `category`: `CONFIG.domain.categories`のいずれか +- `relevance`: 0から5の整数 +- `priority`: A、B、C、Dのいずれか +- `is_corresponding`: 真偽値 +- `pdf_path`: PDFルートからの相対パス + +## 11. 禁止事項 + +- `papers.db`の削除 +- 場当たり的なスキーマ変更 +- PDFの削除、移動、改名 +- DOIや論文内容の推測 +- Orgをマスターとして扱う +- 人が入力したメモの削除 +- `authors`や`keywords`を文字列として出力すること +- `pdf_url`を推測で生成すること diff --git a/paper_db_common/README.org b/paper_db_common/README.org new file mode 100644 index 0000000..ba0957e --- /dev/null +++ b/paper_db_common/README.org @@ -0,0 +1,75 @@ +#+TITLE: paper_db_common + +* 概要 + +複数分野の論文データベースで共有するコード一式である。 +各DBプロジェクトは任意の場所に配置できる。 + +共通側には次を置く。 + +#+begin_example +paper_db_common/ +├── CLAUDE.md +├── new_project.py +├── paperdb_config.py +├── project_loader.py +├── publish.py +├── scripts/ +└── templates/ +#+end_example + +各DB側には分野固有の設定とデータだけを置く。 + +#+begin_example +wglass_db/ +├── config.py +├── papers.db +├── papers.org +├── papers.publish.db +├── logs/ +└── tmp/ +#+end_example + +各DB側に =CLAUDE.md= は不要である。 +=run_ai.py= が共通側の =CLAUDE.md= を読み込み、 +分野固有の条件は =config.py= からプロンプトへ追加する。 + +* 新規DBの作成 + +#+begin_src sh +/path/to/paper_db_common/new_project.py /path/to/new_db +#+end_src + +既存の空でないディレクトリへ作成する場合だけ +=--force= を指定する。 + +* 実行モード + +=publish.py= は実行モードの指定を必須とする。 + +- =process= :: PDF抽出、AI解析、JSON登録によるDB更新 +- =sync= :: PDFパス更新、DB検証、papers.org出力 +- =publish= :: PDF同期、公開用DB作成、リモート反映 +- =all= :: process、sync、publishを順番に実行 + +#+begin_src sh +cd /path/to/new_db +/path/to/paper_db_common/publish.py process +/path/to/paper_db_common/publish.py sync +/path/to/paper_db_common/publish.py publish +/path/to/paper_db_common/publish.py all +#+end_src + +別の場所から実行する場合は =--config= を指定する。 + +#+begin_src sh +/path/to/paper_db_common/publish.py process \ + --config /path/to/new_db/config.py +#+end_src + +* パスの扱い + +各DBの =config.py= に共通コードの絶対パスや +=COMMON_ROOT=、=sys.path= 操作を書く必要はない。 +共通側は =__file__= から自身の位置を決定する。 +=config.py= のディレクトリが =project_root= となる。 diff --git a/paper_db_common/new_project.py b/paper_db_common/new_project.py new file mode 100755 index 0000000..74e2932 --- /dev/null +++ b/paper_db_common/new_project.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""共通テンプレートから新しい論文DBプロジェクトを作成する。""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +COMMON_ROOT = Path(__file__).resolve().parent +TEMPLATE_DIR = COMMON_ROOT / "templates" / "project" +TEMPLATE_FILES = ( + "config.py", + "README.org", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="新しい分野DBを作成する" + ) + parser.add_argument( + "directory", + type=Path, + help="作成するプロジェクトディレクトリ", + ) + parser.add_argument( + "--force", + action="store_true", + help="既存ファイルを上書きする", + ) + return parser.parse_args() + + +def validate_destination( + destination: Path, + force: bool, +) -> None: + if not destination.exists(): + return + if not destination.is_dir(): + raise NotADirectoryError(destination) + if any(destination.iterdir()) and not force: + raise FileExistsError( + f"空でないディレクトリです: {destination}" + ) + + +def copy_template_files(destination: Path) -> None: + for name in TEMPLATE_FILES: + source = TEMPLATE_DIR / name + if not source.is_file(): + raise FileNotFoundError(source) + shutil.copy2(source, destination / name) + + +def create_directories(destination: Path) -> None: + (destination / "tmp" / "text").mkdir( + parents=True, + exist_ok=True, + ) + (destination / "logs").mkdir( + parents=True, + exist_ok=True, + ) + + +def main() -> int: + args = parse_args() + destination = args.directory.expanduser().resolve() + validate_destination(destination, args.force) + destination.mkdir(parents=True, exist_ok=True) + copy_template_files(destination) + create_directories(destination) + print(destination) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/paperdb_config.py b/paper_db_common/paperdb_config.py new file mode 100644 index 0000000..5b7501d --- /dev/null +++ b/paper_db_common/paperdb_config.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class PathConfig: + project_root: Path + pdf_root: Path + database: Path + publish_database: Path + tmp_dir: Path + text_dir: Path + log_dir: Path + papers_org: Path + + +@dataclass(frozen=True) +class RemoteConfig: + host: str + database_dir: str + pdf_dir: str + + +@dataclass(frozen=True) +class AIConfig: + backend: str + claude_permission_mode: str + claude_allowed_tools: tuple[str, ...] + codex_model: str | None + codex_sandbox: str + codex_full_auto: bool + + @property + def claude_allowed_tools_argument(self) -> str: + return ",".join(self.claude_allowed_tools) + + +@dataclass(frozen=True) +class ProcessingConfig: + scan_directories: tuple[str, ...] + ignore_directories: tuple[str, ...] + max_papers: int + initial_pages: int + fallback_pages: int + min_text_length: int + update_database: bool + validate_database: bool + export_org: bool + backup_database: bool + + +@dataclass(frozen=True) +class DomainConfig: + name: str + description: str + categories: tuple[str, ...] + default_category: str + summary_length_ja: str + summary_focus: tuple[str, ...] + relevance_criteria: tuple[str, ...] + priorities: tuple[str, ...] = ("A", "B", "C", "D") + + def validate(self) -> None: + if not self.name.strip(): + raise ValueError("domain.nameが空です") + if not self.description.strip(): + raise ValueError("domain.descriptionが空です") + if not self.categories: + raise ValueError("domain.categoriesが空です") + if len(set(self.categories)) != len(self.categories): + raise ValueError("domain.categoriesに重複があります") + if self.default_category not in self.categories: + raise ValueError( + "default_categoryをcategoriesに含めてください" + ) + if len(self.relevance_criteria) != 6: + raise ValueError( + "relevance_criteriaは0から5の6項目にしてください" + ) + + def categories_prompt(self) -> str: + return "\n".join( + f"- {category}" for category in self.categories + ) + + def summary_focus_prompt(self) -> str: + return "\n".join( + f"- {item}" for item in self.summary_focus + ) + + def relevance_prompt(self) -> str: + return "\n".join( + f"- {score}: {description}" + for score, description in enumerate( + self.relevance_criteria + ) + ) + + +@dataclass(frozen=True) +class Config: + path: PathConfig + remote: RemoteConfig + ai: AIConfig + processing: ProcessingConfig + domain: DomainConfig + + @property + def scan_paths(self) -> tuple[Path, ...]: + return tuple( + self.path.pdf_root / directory + for directory in self.processing.scan_directories + ) + + def validate(self, *, require_remote: bool = False) -> None: + root = self.path.project_root.resolve() + if not root.is_dir(): + raise NotADirectoryError(root) + if not self.path.pdf_root.is_dir(): + raise NotADirectoryError( + f"PDF_ROOTがありません: {self.path.pdf_root}" + ) + if not self.processing.scan_directories: + raise ValueError("scan_directoriesが空です") + for path in self.scan_paths: + if not path.is_dir(): + raise NotADirectoryError( + f"探索ディレクトリがありません: {path}" + ) + if self.processing.max_papers < 1: + raise ValueError("max_papersは1以上にしてください") + if self.processing.initial_pages < 1: + raise ValueError("initial_pagesは1以上にしてください") + if ( + self.processing.fallback_pages + < self.processing.initial_pages + ): + raise ValueError( + "fallback_pagesはinitial_pages以上にしてください" + ) + if self.processing.min_text_length < 0: + raise ValueError( + "min_text_lengthは0以上にしてください" + ) + if self.ai.backend.lower() not in {"claude", "codex"}: + raise ValueError( + "ai.backendはclaudeまたはcodexにしてください" + ) + self.domain.validate() + if require_remote: + values = ( + self.remote.host, + self.remote.database_dir, + self.remote.pdf_dir, + ) + if any(not value.strip() for value in values): + raise ValueError("公開サーバ設定が不足しています") diff --git a/paper_db_common/project_loader.py b/paper_db_common/project_loader.py new file mode 100644 index 0000000..13e2e17 --- /dev/null +++ b/paper_db_common/project_loader.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +ENV_NAME = "PAPER_DB_CONFIG" + +COMMON_ROOT = Path(__file__).resolve().parent +if str(COMMON_ROOT) not in sys.path: + sys.path.insert(0, str(COMMON_ROOT)) + + +def resolve_config_path(value: str | Path | None = None) -> Path: + raw = value or os.environ.get(ENV_NAME) + path = Path(raw).expanduser() if raw else Path.cwd() / "config.py" + path = path.resolve() + if not path.is_file(): + raise FileNotFoundError(f"config.pyがありません: {path}") + return path + + +def load_module(path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location( + "paper_db_project_config", path + ) + if spec is None or spec.loader is None: + raise ImportError(f"config.pyを読み込めません: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_config(value: str | Path | None = None) -> tuple[Path, Any]: + path = resolve_config_path(value) + module = load_module(path) + if not hasattr(module, "CONFIG"): + raise AttributeError(f"CONFIGがありません: {path}") + config = module.CONFIG + if Path(config.path.project_root).resolve() != path.parent: + raise ValueError( + "project_rootはconfig.pyのディレクトリにしてください" + ) + return path, config diff --git a/paper_db_common/publish.py b/paper_db_common/publish.py new file mode 100755 index 0000000..426d1c1 --- /dev/null +++ b/paper_db_common/publish.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import os +import shlex +import shutil +import sqlite3 +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + +from project_loader import load_config + +COMMON_ROOT = Path(__file__).resolve().parent +SCRIPTS_DIR = COMMON_ROOT / "scripts" + + +@dataclass(frozen=True) +class RuntimeOptions: + mode: str + backend: str + max_papers: int + + +def positive_int(value: str) -> int: + number = int(value) + if number < 1: + raise argparse.ArgumentTypeError("1以上を指定してください") + return number + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="論文解析、DB整理、公開処理を実行する" + ) + parser.add_argument( + "mode", + choices=("process", "sync", "publish", "all"), + help="実行モード", + ) + parser.add_argument( + "--config", + help="省略時はカレントディレクトリのconfig.py", + ) + parser.add_argument( + "--backend", + choices=("claude", "codex"), + help=( + "使用するAIバックエンド。省略時は" + "config.pyの設定を使用" + ), + ) + parser.add_argument( + "--max-papers", + type=positive_int, + metavar="N", + help=( + "1回に処理する最大論文数。省略時は" + "config.pyの設定を使用" + ), + ) + return parser.parse_args() + + +def run_command( + command: Sequence[object], + *, + config_path: Path, + config: object, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + args = [str(item) for item in command] + print("+", " ".join(args)) + env = os.environ.copy() + env["PAPER_DB_CONFIG"] = str(config_path) + return subprocess.run( + args, + cwd=config.path.project_root, + env=env, + text=True, + check=check, + ) + + +def require_commands(names: Sequence[str]) -> None: + for name in names: + if shutil.which(name) is None: + raise RuntimeError( + f"必要なコマンドが見つかりません: {name}" + ) + + +def script(name: str) -> Path: + path = SCRIPTS_DIR / name + if not path.is_file(): + raise FileNotFoundError(path) + return path + + +def ensure_database(config_path: Path, config: object) -> None: + if config.path.database.is_file(): + return + print("=== データベース初期化 ===") + run_command( + [sys.executable, script("init_db.py"), config.path.database], + config_path=config_path, + config=config, + ) + + +def prepare_work_directories(config: object) -> None: + shutil.rmtree(config.path.tmp_dir, ignore_errors=True) + config.path.text_dir.mkdir(parents=True, exist_ok=True) + config.path.log_dir.mkdir(parents=True, exist_ok=True) + + +def extract_text( + config_path: Path, config: object, max_papers: int +) -> bool: + p = config.processing + command: list[object] = [ + sys.executable, script("extract_text.py"), + "--database", config.path.database, + "--pdf-root", config.path.pdf_root, + "--output-dir", config.path.text_dir, + "--limit", max_papers, + "--pages", p.initial_pages, + "--fallback-pages", p.fallback_pages, + "--min-text-length", p.min_text_length, + ] + for directory in p.scan_directories: + command.extend(["--scan-dir", directory]) + result = run_command( + command, + config_path=config_path, + config=config, + check=False, + ) + if result.returncode == 0: + return True + if result.returncode == 10: + print("=== 未登録PDFはありません ===") + return False + raise RuntimeError( + f"extract_text.pyが失敗しました: {result.returncode}" + ) + + +def process_new_papers( + config_path: Path, config: object, options: RuntimeOptions +) -> None: + config.validate() + require_commands(("pdftotext", options.backend)) + ensure_database(config_path, config) + prepare_work_directories(config) + if not extract_text(config_path, config, options.max_papers): + return + run_command( + [sys.executable, script("run_ai.py"), + "--backend", options.backend, + "--max-papers", options.max_papers], + config_path=config_path, + config=config, + ) + if config.processing.update_database: + run_command( + [sys.executable, script("update_all.py"), + "--input-dir", config.path.tmp_dir, + "--database", config.path.database, + "--pdf-root", config.path.pdf_root, + "--continue-on-error"], + config_path=config_path, + config=config, + ) + + +def sync_local_database(config_path: Path, config: object) -> None: + """PDFパス更新、DB検証、Org出力を実行する。""" + config.validate() + ensure_database(config_path, config) + + run_command( + [ + sys.executable, + script("update_pdf_paths.py"), + "--database", + config.path.database, + "--pdf-root", + config.path.pdf_root, + "--apply", + ], + config_path=config_path, + config=config, + ) + + if config.processing.validate_database: + run_command( + [ + sys.executable, + script("validate_db.py"), + config.path.database, + "--pdf-root", + config.path.pdf_root, + ], + config_path=config_path, + config=config, + ) + + if config.processing.export_org: + run_command( + [ + sys.executable, + script("export_org.py"), + config.path.database, + "-o", + config.path.papers_org, + ], + config_path=config_path, + config=config, + ) + + +def publish_remote(config_path: Path, config: object) -> None: + """PDFと公開用DBをリモートへ反映する。""" + config.validate(require_remote=True) + require_commands(("rsync", "scp", "ssh")) + if not config.path.database.is_file(): + raise FileNotFoundError(config.path.database) + sync_pdfs(config_path, config) + publish_database(config_path, config) + + +def create_publish_database(config: object) -> None: + config.path.publish_database.unlink(missing_ok=True) + with sqlite3.connect(config.path.database) as source: + with sqlite3.connect(config.path.publish_database) as target: + source.backup(target) + with sqlite3.connect(config.path.publish_database) as con: + con.execute("PRAGMA journal_mode=DELETE") + con.commit() + + +def shell_quote(value: str) -> str: + return shlex.quote(value) + + +def publish_database(config_path: Path, config: object) -> None: + config.validate(require_remote=True) + require_commands(("scp", "ssh")) + if not config.path.database.is_file(): + raise FileNotFoundError(config.path.database) + create_publish_database(config) + destination = ( + f"{config.remote.host}:" + f"{config.remote.database_dir}/papers.db.new" + ) + run_command( + ["scp", config.path.publish_database, destination], + config_path=config_path, + config=config, + ) + remote_dir = shell_quote(config.remote.database_dir) + remote_script = f"""set -eu +cd {remote_dir} +sqlite3 papers.db.new 'PRAGMA integrity_check;' +mv -f papers.db.new papers.db +sqlite3 papers.db 'PRAGMA integrity_check;' +""" + run_command( + ["ssh", config.remote.host, remote_script], + config_path=config_path, + config=config, + ) + + +def sync_pdfs(config_path: Path, config: object) -> None: + config.validate(require_remote=True) + require_commands(("rsync",)) + run_command( + ["rsync", "-az", f"{config.path.pdf_root}/", + f"{config.remote.host}:{config.remote.pdf_dir}/"], + config_path=config_path, + config=config, + ) + + +def main() -> int: + args = parse_args() + try: + config_path, config = load_config(args.config) + options = RuntimeOptions( + mode=args.mode, + backend=args.backend or config.ai.backend, + max_papers=( + args.max_papers + or config.processing.max_papers + ), + ) + + if options.mode in {"process", "all"}: + process_new_papers(config_path, config, options) + + if options.mode in {"sync", "all"}: + sync_local_database(config_path, config) + + if options.mode in {"publish", "all"}: + publish_remote(config_path, config) + + print("=== 完了 ===") + return 0 + except KeyboardInterrupt: + print("中断しました。", file=sys.stderr) + return 130 + except Exception as exc: + print(f"publish.py: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/common.py b/paper_db_common/scripts/common.py new file mode 100755 index 0000000..fa7f7c5 --- /dev/null +++ b/paper_db_common/scripts/common.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import hashlib +import html +import re +import unicodedata +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import sys + +COMMON_ROOT = Path(__file__).resolve().parents[1] +if str(COMMON_ROOT) not in sys.path: + sys.path.insert(0, str(COMMON_ROOT)) + +from project_loader import load_config # noqa: E402 + +_, CONFIG = load_config() +ALLOWED_PRIORITIES = set(CONFIG.domain.priorities) +ALLOWED_CATEGORIES = set(CONFIG.domain.categories) +def now_iso() -> str: + return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + +def normalize_doi(value: Any) -> str | None: + if value is None: + return None + doi = str(value).strip().lower() + if not doi or doi == "unknown": + return None + prefixes = ( + "https://doi.org/", + "http://doi.org/", + "http://dx.doi.org/", + "doi:", + ) + for prefix in prefixes: + if doi.startswith(prefix): + doi = doi[len(prefix):] + break + doi = doi.strip().rstrip(".,;:)]}") + return doi or None + +def normalize_text(value: str) -> str: + value = html.unescape(value) + value = unicodedata.normalize("NFKC", value) + value = value.lower() + value = re.sub(r"[\r\n\t]+", " ", value) + value = re.sub(r"[^\w\s]", " ", value, flags=re.UNICODE) + value = re.sub(r"\s+", " ", value).strip() + return value + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for block in iter(lambda: f.read(1024 * 1024), b""): + h.update(block) + return h.hexdigest() + +def resolve_pdf_path(pdf_root: Path, stored_path: str) -> Path: + path = Path(stored_path) + if path.is_absolute(): + raise ValueError("pdf_path must be relative to PDF_ROOT") + resolved = (pdf_root / path).resolve() + root = pdf_root.resolve() + try: + resolved.relative_to(root) + except ValueError as exc: + raise ValueError("pdf_path escapes PDF_ROOT") from exc + return resolved diff --git a/paper_db_common/scripts/delete_paper.py b/paper_db_common/scripts/delete_paper.py new file mode 100755 index 0000000..d1d44a6 --- /dev/null +++ b/paper_db_common/scripts/delete_paper.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""paper_idを指定してpapers.dbから論文を削除する。 + +PDFファイルそのものは削除しない。 +既定はdry-run。実際に削除する場合は--applyを指定する。 +""" + +from __future__ import annotations + +import argparse +import shutil +import sqlite3 +import sys +from dataclasses import dataclass +from pathlib import Path + +from common import CONFIG + + +@dataclass(frozen=True) +class Paper: + paper_id: int + title: str + first_author: str | None + year: int | None + doi: str | None + pdf_path: str | None + file_sha256: str | None + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="paper_idを指定してDBレコードを削除する" + ) + parser.add_argument( + "paper_id", + type=int, + help="削除するpapers.id", + ) + parser.add_argument( + "--database", + type=Path, + default=CONFIG.path.database, + help="SQLiteデータベース", + ) + parser.add_argument( + "--apply", + action="store_true", + help="削除を実際にDBへ反映する", + ) + parser.add_argument( + "--no-backup", + action="store_true", + help="削除前バックアップを作成しない", + ) + return parser.parse_args() + + +def open_database( + database: Path, +) -> sqlite3.Connection: + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys=ON") + return connection + + +def load_paper( + connection: sqlite3.Connection, + paper_id: int, +) -> Paper | None: + row = connection.execute( + """ + SELECT + id, + title, + first_author, + year, + doi, + pdf_path, + file_sha256 + FROM papers + WHERE id = ? + """, + (paper_id,), + ).fetchone() + + if row is None: + return None + + return Paper( + paper_id=row["id"], + title=row["title"], + first_author=row["first_author"], + year=row["year"], + doi=row["doi"], + pdf_path=row["pdf_path"], + file_sha256=row["file_sha256"], + ) + + +def print_paper(paper: Paper) -> None: + print("=== 削除対象 ===") + print(f"id: {paper.paper_id}") + print(f"title: {paper.title}") + print( + "first_author: " + f"{paper.first_author or '(none)'}" + ) + print(f"year: {paper.year or '(none)'}") + print(f"doi: {paper.doi or '(none)'}") + print( + f"pdf_path: {paper.pdf_path or '(none)'}" + ) + print( + "file_sha256: " + f"{paper.file_sha256 or '(none)'}" + ) + + +def create_backup(database: Path) -> Path: + backup = database.with_name( + database.name + ".before-delete" + ) + + counter = 1 + + while backup.exists(): + backup = database.with_name( + database.name + f".before-delete.{counter}" + ) + counter += 1 + + shutil.copy2(database, backup) + return backup + + +def delete_paper( + connection: sqlite3.Connection, + paper_id: int, +) -> None: + """関連レコードを含めて1トランザクションで削除する。""" + with connection: + tables = ( + "paper_authors", + "paper_keywords", + "processing_log", + ) + + for table in tables: + if table_exists(connection, table): + connection.execute( + f"DELETE FROM {table} WHERE paper_id = ?", + (paper_id,), + ) + + cursor = connection.execute( + "DELETE FROM papers WHERE id = ?", + (paper_id,), + ) + + if cursor.rowcount != 1: + raise RuntimeError( + f"papers.id={paper_id}を削除できませんでした" + ) + + +def table_exists( + connection: sqlite3.Connection, + table: str, +) -> bool: + row = connection.execute( + """ + SELECT 1 + FROM sqlite_master + WHERE type = 'table' + AND name = ? + """, + (table,), + ).fetchone() + + return row is not None + + +def validate_database( + connection: sqlite3.Connection, +) -> None: + integrity = connection.execute( + "PRAGMA integrity_check" + ).fetchone()[0] + + if integrity != "ok": + raise RuntimeError( + f"integrity_check: {integrity}" + ) + + foreign_keys = connection.execute( + "PRAGMA foreign_key_check" + ).fetchall() + + if foreign_keys: + raise RuntimeError( + "foreign_key_check: " + f"{len(foreign_keys)}件" + ) + + +def main() -> int: + args = parse_args() + + try: + database = args.database.expanduser().resolve() + + if not database.is_file(): + raise FileNotFoundError(database) + + with open_database(database) as connection: + paper = load_paper( + connection, + args.paper_id, + ) + + if paper is None: + print( + f"paper_id={args.paper_id}は存在しません。", + file=sys.stderr, + ) + return 1 + + print_paper(paper) + + if not args.apply: + print() + print( + "削除するには --apply を指定してください。" + ) + return 0 + + if not args.no_backup: + backup = create_backup(database) + print() + print(f"DBバックアップ: {backup}") + + delete_paper( + connection, + args.paper_id, + ) + validate_database(connection) + + print() + print( + f"Deleted paper_id={args.paper_id}" + ) + print("PDFファイルは削除していません。") + return 0 + + except KeyboardInterrupt: + return 130 + + except Exception as exc: + print( + f"delete_paper.py: {exc}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/export_org.py b/paper_db_common/scripts/export_org.py new file mode 100755 index 0000000..f86fcb9 --- /dev/null +++ b/paper_db_common/scripts/export_org.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sqlite3 +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Sequence, TextIO + +from common import CONFIG + + +def prop(value: object) -> str: + if value is None: + return "" + return str(value).replace("\n", " ").strip() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "database", + nargs="?", + default=str(CONFIG.path.database), + ) + parser.add_argument( + "-o", + "--output", + help="出力先。省略時は標準出力", + ) + return parser.parse_args() + + +def open_database(database: str) -> sqlite3.Connection: + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + return connection + + +def load_papers( + connection: sqlite3.Connection, +) -> list[sqlite3.Row]: + query = """ + SELECT p.*, c.name AS category + FROM papers AS p + LEFT JOIN categories AS c + ON c.id = p.category_id + ORDER BY + COALESCE(p.year, 0) DESC, + p.first_author, + p.title + """ + return connection.execute(query).fetchall() + + +def load_authors( + connection: sqlite3.Connection, + paper_id: int, +) -> list[str]: + query = """ + SELECT a.name + FROM paper_authors AS pa + JOIN authors AS a + ON a.id = pa.author_id + WHERE pa.paper_id = ? + ORDER BY pa.author_order + """ + rows = connection.execute(query, (paper_id,)).fetchall() + return [row["name"] for row in rows] + + +def load_keywords( + connection: sqlite3.Connection, + paper_id: int, +) -> list[str]: + query = """ + SELECT k.name + FROM paper_keywords AS pk + JOIN keywords AS k + ON k.id = pk.keyword_id + WHERE pk.paper_id = ? + ORDER BY k.name_normalized + """ + rows = connection.execute(query, (paper_id,)).fetchall() + return [row["name"] for row in rows] + + +def make_header() -> list[str]: + exported_at = ( + datetime.now(timezone.utc) + .astimezone() + .isoformat(timespec="seconds") + ) + return [ + "#+TITLE: ガラス固化体論文データベース", + f"#+DB_EXPORTED_AT: {exported_at}", + "", + ( + "# 編集可能: CATEGORY, RELEVANCE, PRIORITY, STATUS, " + "日本語要約, キーワード, メモ" + ), + ( + "# 保護対象: DB_ID, DOI, TITLE, AUTHORS, YEAR, " + "JOURNAL, PDF_PATH, FILE_SHA256" + ), + "", + ] + + +def make_properties( + paper: sqlite3.Row, + authors: Sequence[str], +) -> list[str]: + category = paper["category"] or "Other" + return [ + ":PROPERTIES:", + f":DB_ID: {paper['id']}", + f":DB_UPDATED_AT: {prop(paper['updated_at'])}", + f":DOI: {prop(paper['doi'])}", + f":TITLE: {prop(paper['title'])}", + f":AUTHORS: {prop('; '.join(authors))}", + f":FIRST_AUTHOR: {prop(paper['first_author'])}", + f":YEAR: {prop(paper['year'])}", + f":JOURNAL: {prop(paper['journal'])}", + f":PDF_PATH: {prop(paper['pdf_path'])}", + f":FILE_SHA256: {prop(paper['file_sha256'])}", + f":CATEGORY: {prop(category)}", + f":RELEVANCE: {prop(paper['relevance'])}", + f":PRIORITY: {prop(paper['priority'])}", + f":STATUS: {prop(paper['review_status'])}", + ":END:", + ] + + +def make_paper_entry( + paper: sqlite3.Row, + authors: Sequence[str], + keywords: Sequence[str], +) -> list[str]: + paper_id = paper["id"] + title = prop(paper["title"]) + lines = [ + f"* [ID:{paper_id}] {title}", + *make_properties(paper, authors), + "", + "** 日本語要約", + prop(paper["summary_ja"]), + "", + "** キーワード", + ] + lines.extend(f"- {keyword}" for keyword in keywords) + lines.extend(["", "** メモ", prop(paper["notes"]), ""]) + return lines + + +def build_org( + connection: sqlite3.Connection, + papers: Sequence[sqlite3.Row], +) -> str: + lines = make_header() + for paper in papers: + paper_id = paper["id"] + authors = load_authors(connection, paper_id) + keywords = load_keywords(connection, paper_id) + lines.extend(make_paper_entry(paper, authors, keywords)) + return "\n".join(lines) + + +def write_output(content: str, output: str | None) -> None: + if output is None: + sys.stdout.write(content) + if not content.endswith("\n"): + sys.stdout.write("\n") + return + + Path(output).write_text(content, encoding="utf-8") + + +def print_summary( + paper_count: int, + output: str | None, + stream: TextIO = sys.stderr, +) -> None: + destination = output if output is not None else "stdout" + print( + f"Exported {paper_count} papers to {destination}", + file=stream, + ) + + +def main() -> int: + args = parse_args() + + with open_database(args.database) as connection: + papers = load_papers(connection) + content = build_org(connection, papers) + + write_output(content, args.output) + print_summary(len(papers), args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/extract_text.py b/paper_db_common/scripts/extract_text.py new file mode 100755 index 0000000..251c98e --- /dev/null +++ b/paper_db_common/scripts/extract_text.py @@ -0,0 +1,1137 @@ +#!/usr/bin/env python3 +""" +未登録PDFを検出し、先頭ページを段階的にテキスト化する。 + +通常は先頭2ページだけを抽出する。 +抽出文字数が --min-text-length 未満の場合は、 +--fallback-pages まで再抽出する。 + +既存PDFと正規化本文のSHA256が一致した場合は、 +重複PDFとして paper_files テーブルへ代替版登録し、 +AI解析用テキストを生成しない。 + +既存PDFの本文ハッシュは paper_files テーブルへ保存する。 +初回実行時は既存PDFのハッシュ作成に時間がかかるが、 +2回目以降はファイルのSHA256が変わらない限り再利用する。 + +例: + python3 scripts/extract_text.py \ + --database papers.db \ + --pdf-root "/path/to/PDF_ROOT" \ + --output-dir tmp/text \ + --limit 3 \ + --pages 2 \ + --fallback-pages 5 \ + --min-text-length 1500 \ + --scan-dir "大窪" \ + --scan-dir "JAEA三ツ井" + +終了コード: + 0 1件以上のテキストを生成、または重複PDFを記録 + 10 未登録PDFなし + 1 エラー +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import unicodedata +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from common import CONFIG + + +MIN_DUPLICATE_TEXT_LENGTH = 500 + + +@dataclass(frozen=True) +class RegisteredPaper: + paper_id: int + pdf_path: str + normalized_path: str + + +@dataclass(frozen=True) +class TextHashRecord: + paper_id: int + pdf_path: str + file_sha256: str + text_sha256: str + text_length: int + + +@dataclass(frozen=True) +class ProcessResult: + generated: bool + duplicate: bool + text_length: int + insufficient: bool + fallback_used: bool + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "papers.dbに未登録のPDFを段階的にテキスト化する" + ) + ) + parser.add_argument( + "--database", + default=CONFIG.path.database, + type=Path, + help="papers.db のパス", + ) + parser.add_argument( + "--pdf-root", + default=CONFIG.path.pdf_root, + type=Path, + help="PDF_ROOT のパス", + ) + parser.add_argument( + "--output-dir", + default=CONFIG.path.text_dir, + type=Path, + help="抽出テキストの出力先", + ) + parser.add_argument( + "--limit", + type=int, + default=CONFIG.processing.max_papers, + help="AI解析用テキストを生成する最大件数", + ) + parser.add_argument( + "--pages", + type=int, + default=CONFIG.processing.initial_pages, + help="最初に抽出する先頭ページ数", + ) + parser.add_argument( + "--fallback-pages", + type=int, + default=CONFIG.processing.fallback_pages, + help=( + "文字数不足時に再抽出する先頭ページ数。" + "0で再抽出しない" + ), + ) + parser.add_argument( + "--min-text-length", + type=int, + default=CONFIG.processing.min_text_length, + help=( + "再抽出を判断する最小文字数。" + "これ未満ならfallback-pagesまで再抽出する" + ), + ) + parser.add_argument( + "--scan-dir", + action="append", + default=None, + help=( + "PDF_ROOTからの相対探索ディレクトリ。" + "複数指定可能。未指定時は設定値を使用" + ), + ) + parser.add_argument( + "--rebuild-text-hashes", + action="store_true", + help="既存PDFの本文ハッシュキャッシュを再作成する", + ) + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> None: + if not args.database.is_file(): + raise FileNotFoundError( + f"データベースがありません: {args.database}" + ) + + if not args.pdf_root.is_dir(): + raise NotADirectoryError( + f"PDF_ROOTがありません: {args.pdf_root}" + ) + + if args.limit < 1: + raise ValueError("--limit は1以上にしてください") + + if args.pages < 1: + raise ValueError("--pages は1以上にしてください") + + if args.fallback_pages < 0: + raise ValueError("--fallback-pages は0以上にしてください") + + if ( + args.fallback_pages > 0 + and args.fallback_pages < args.pages + ): + raise ValueError( + "--fallback-pages は --pages 以上にしてください" + ) + + if args.min_text_length < 0: + raise ValueError( + "--min-text-length は0以上にしてください" + ) + + if shutil.which("pdftotext") is None: + raise RuntimeError( + "pdftotext が見つかりません。" + "macOSでは `sudo port install poppler` " + "または `brew install poppler` で導入できます。" + ) + + +def normalize_pdf_path(value: str) -> str: + """PDFパスを比較用に正規化する。""" + normalized = unicodedata.normalize("NFC", value) + normalized = normalized.replace("\\", "/") + normalized = normalized.lstrip("./") + return normalized.casefold() + + +def normalize_text(value: str) -> str: + """本文を重複判定用に正規化する。""" + normalized = unicodedata.normalize("NFKC", value) + normalized = normalized.replace("\x00", "") + normalized = re.sub(r"\s+", " ", normalized) + return normalized.strip() + + +def calculate_text_sha256(value: str) -> str: + """正規化本文のSHA256を返す。""" + normalized = normalize_text(value) + return hashlib.sha256( + normalized.encode("utf-8") + ).hexdigest() + + +def ensure_paper_files_table(conn: sqlite3.Connection) -> None: + """paper_filesを作成し、旧データを自動移行する。""" + conn.execute( + """ + CREATE TABLE IF NOT EXISTS paper_files ( + id INTEGER PRIMARY KEY, + paper_id INTEGER NOT NULL, + pdf_path TEXT NOT NULL UNIQUE, + file_sha256 TEXT, + text_sha256 TEXT, + text_length INTEGER, + version_type TEXT NOT NULL DEFAULT 'unknown' + CHECK (version_type IN ( + 'publisher', + 'author', + 'repository', + 'scan', + 'translated', + 'duplicate', + 'unknown' + )), + is_primary INTEGER NOT NULL DEFAULT 0 + CHECK (is_primary IN (0, 1)), + notes TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (paper_id) + REFERENCES papers(id) + ON DELETE CASCADE + ) + """ + ) + conn.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS + idx_paper_files_one_primary + ON paper_files(paper_id) + WHERE is_primary = 1 + """ + ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS + idx_paper_files_text_sha256 + ON paper_files(text_sha256) + """ + ) + conn.execute( + """ + CREATE INDEX IF NOT EXISTS + idx_paper_files_file_sha256 + ON paper_files(file_sha256) + """ + ) + + # papers.pdf_pathを主PDFとして移行する。 + conn.execute( + """ + INSERT OR IGNORE INTO paper_files ( + paper_id, + pdf_path, + file_sha256, + version_type, + is_primary, + notes + ) + SELECT + id, + pdf_path, + file_sha256, + 'unknown', + 1, + 'papers.pdf_pathから移行' + FROM papers + WHERE pdf_path IS NOT NULL + AND TRIM(pdf_path) <> '' + """ + ) + + # 旧本文ハッシュキャッシュを取り込む。 + if table_exists(conn, "pdf_text_hashes"): + conn.execute( + """ + UPDATE paper_files + SET + file_sha256 = COALESCE( + ( + SELECT h.file_sha256 + FROM pdf_text_hashes AS h + WHERE h.pdf_path = paper_files.pdf_path + ), + file_sha256 + ), + text_sha256 = COALESCE( + ( + SELECT h.text_sha256 + FROM pdf_text_hashes AS h + WHERE h.pdf_path = paper_files.pdf_path + ), + text_sha256 + ), + text_length = COALESCE( + ( + SELECT h.text_length + FROM pdf_text_hashes AS h + WHERE h.pdf_path = paper_files.pdf_path + ), + text_length + ) + WHERE EXISTS ( + SELECT 1 + FROM pdf_text_hashes AS h + WHERE h.pdf_path = paper_files.pdf_path + ) + """ + ) + + # 旧duplicate_filesを代替PDFとして移行する。 + if table_exists(conn, "duplicate_files"): + conn.execute( + """ + INSERT OR IGNORE INTO paper_files ( + paper_id, + pdf_path, + file_sha256, + text_sha256, + version_type, + is_primary, + notes + ) + SELECT + paper_id, + pdf_path, + file_sha256, + text_sha256, + 'duplicate', + 0, + '旧duplicate_filesから移行; matched_pdf_path=' || + matched_pdf_path + FROM duplicate_files + """ + ) + + +def table_exists( + conn: sqlite3.Connection, + table_name: str, +) -> bool: + row = conn.execute( + """ + SELECT 1 + FROM sqlite_master + WHERE type = 'table' + AND name = ? + """, + (table_name,), + ).fetchone() + return row is not None + + +def load_registered_papers( + conn: sqlite3.Connection, +) -> list[RegisteredPaper]: + """paper_filesに登録されたすべてのPDFを返す。""" + rows = conn.execute( + """ + SELECT paper_id, pdf_path + FROM paper_files + WHERE pdf_path IS NOT NULL + AND TRIM(pdf_path) <> '' + ORDER BY paper_id, is_primary DESC, id + """ + ).fetchall() + + return [ + RegisteredPaper( + paper_id=int(row[0]), + pdf_path=str(row[1]), + normalized_path=normalize_pdf_path(str(row[1])), + ) + for row in rows + ] + + +def load_known_paths( + conn: sqlite3.Connection, +) -> set[str]: + rows = conn.execute( + """ + SELECT pdf_path + FROM paper_files + WHERE pdf_path IS NOT NULL + AND TRIM(pdf_path) <> '' + """ + ).fetchall() + + return { + normalize_pdf_path(str(row[0])) + for row in rows + if row[0] is not None + } + + +def load_cached_text_hash( + conn: sqlite3.Connection, + pdf_path: str, +) -> TextHashRecord | None: + row = conn.execute( + """ + SELECT + paper_id, + pdf_path, + file_sha256, + text_sha256, + text_length + FROM paper_files + WHERE pdf_path = ? + AND file_sha256 IS NOT NULL + AND text_sha256 IS NOT NULL + AND text_length IS NOT NULL + """, + (pdf_path,), + ).fetchone() + + if row is None: + return None + + return TextHashRecord( + paper_id=int(row[0]), + pdf_path=str(row[1]), + file_sha256=str(row[2]), + text_sha256=str(row[3]), + text_length=int(row[4]), + ) + + +def save_text_hash( + conn: sqlite3.Connection, + record: TextHashRecord, +) -> None: + conn.execute( + """ + UPDATE paper_files + SET + file_sha256 = ?, + text_sha256 = ?, + text_length = ?, + updated_at = CURRENT_TIMESTAMP + WHERE pdf_path = ? + """, + ( + record.file_sha256, + record.text_sha256, + record.text_length, + record.pdf_path, + ), + ) + + +def save_duplicate_file( + conn: sqlite3.Connection, + paper_id: int, + pdf_path: str, + file_sha256: str, + text_sha256: str, + matched_pdf_path: str, +) -> None: + """重複PDFを同じ論文の代替ファイルとして登録する。""" + conn.execute( + """ + INSERT INTO paper_files ( + paper_id, + pdf_path, + file_sha256, + text_sha256, + version_type, + is_primary, + notes, + created_at, + updated_at + ) + VALUES ( + ?, ?, ?, ?, 'duplicate', 0, ?, + CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP + ) + ON CONFLICT(pdf_path) DO UPDATE SET + paper_id = excluded.paper_id, + file_sha256 = excluded.file_sha256, + text_sha256 = excluded.text_sha256, + version_type = excluded.version_type, + is_primary = 0, + notes = excluded.notes, + updated_at = CURRENT_TIMESTAMP + """, + ( + paper_id, + pdf_path, + file_sha256, + text_sha256, + f"本文一致: {matched_pdf_path}", + ), + ) + +def iter_scan_roots( + pdf_root: Path, + scan_dirs: list[str], +) -> list[Path]: + if not scan_dirs: + return [pdf_root] + + roots: list[Path] = [] + + for value in scan_dirs: + relative = Path(value) + + if relative.is_absolute(): + raise ValueError( + "--scan-dir はPDF_ROOTからの相対パスに" + f"してください: {value}" + ) + + root = (pdf_root / relative).resolve() + + try: + root.relative_to(pdf_root.resolve()) + except ValueError as exc: + raise ValueError( + f"PDF_ROOT外のディレクトリは探索できません: " + f"{value}" + ) from exc + + if not root.is_dir(): + raise NotADirectoryError( + f"探索ディレクトリがありません: {root}" + ) + + roots.append(root) + + return roots + + +def iter_pdf_files( + scan_roots: Iterable[Path], +) -> Iterable[Path]: + found: set[Path] = set() + + for root in scan_roots: + for pdf in root.rglob("*"): + if not pdf.is_file(): + continue + if pdf.suffix.lower() != ".pdf": + continue + + resolved = pdf.resolve() + + if resolved in found: + continue + + found.add(resolved) + yield resolved + + +def calculate_sha256(path: Path) -> str: + digest = hashlib.sha256() + + with path.open("rb") as file: + for chunk in iter( + lambda: file.read(1024 * 1024), + b"", + ): + digest.update(chunk) + + return digest.hexdigest() + + +def read_text(path: Path) -> str: + return path.read_text( + encoding="utf-8", + errors="replace", + ).strip() + + +def run_pdftotext( + pdf_path: Path, + output_path: Path, + pages: int, +) -> None: + command = [ + "pdftotext", + "-enc", + "UTF-8", + "-f", + "1", + "-l", + str(pages), + str(pdf_path), + str(output_path), + ] + + completed = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + + if completed.returncode != 0: + error = ( + completed.stderr.strip() + or "不明なpdftotextエラー" + ) + raise RuntimeError(error) + + +def extract_pdf_text( + pdf_path: Path, + output_path: Path, + pages: int, + fallback_pages: int, + min_text_length: int, +) -> tuple[str, int, bool]: + run_pdftotext( + pdf_path=pdf_path, + output_path=output_path, + pages=pages, + ) + + body = read_text(output_path) + used_pages = pages + fallback_used = False + + should_retry = ( + fallback_pages > pages + and len(body) < min_text_length + ) + + if should_retry: + run_pdftotext( + pdf_path=pdf_path, + output_path=output_path, + pages=fallback_pages, + ) + body = read_text(output_path) + used_pages = fallback_pages + fallback_used = True + + return body, used_pages, fallback_used + + +def extract_text_to_memory( + pdf_path: Path, + pages: int, + fallback_pages: int, + min_text_length: int, +) -> tuple[str, int, bool]: + with tempfile.TemporaryDirectory() as tmp_dir: + output_path = Path(tmp_dir) / "text.txt" + return extract_pdf_text( + pdf_path=pdf_path, + output_path=output_path, + pages=pages, + fallback_pages=fallback_pages, + min_text_length=min_text_length, + ) + + +def write_text_with_metadata( + text_path: Path, + body: str, + relative_pdf_path: str, + file_sha256: str, + text_sha256: str, + file_size: int, + requested_pages: int, + extracted_pages: int, + fallback_used: bool, + min_text_length: int, +) -> int: + text_length = len(body) + pdf_text_empty = "yes" if text_length == 0 else "no" + text_insufficient = ( + "yes" + if text_length < min_text_length + else "no" + ) + fallback_value = "yes" if fallback_used else "no" + + header = ( + f"PDF_PATH: {relative_pdf_path}\n" + f"FILE_SHA256: {file_sha256}\n" + f"TEXT_SHA256: {text_sha256}\n" + f"PDF_FILE_SIZE: {file_size}\n" + f"REQUESTED_PAGES: {requested_pages}\n" + f"EXTRACTED_PAGES: {extracted_pages}\n" + f"FALLBACK_USED: {fallback_value}\n" + f"TEXT_LENGTH: {text_length}\n" + f"PDF_TEXT_EMPTY: {pdf_text_empty}\n" + f"TEXT_INSUFFICIENT: {text_insufficient}\n" + "\n" + "----- BEGIN PDF TEXT -----\n" + "\n" + ) + + footer = "\n\n----- END PDF TEXT -----\n" + + text_path.write_text( + header + body + footer, + encoding="utf-8", + ) + + return text_length + + +def collect_candidates( + pdf_root: Path, + scan_roots: Iterable[Path], + known_paths: set[str], +) -> list[tuple[str, Path]]: + candidates: list[tuple[str, Path]] = [] + + for pdf_path in iter_pdf_files(scan_roots): + relative_path = pdf_path.relative_to( + pdf_root + ).as_posix() + + if normalize_pdf_path(relative_path) in known_paths: + continue + + candidates.append( + (relative_path, pdf_path) + ) + + candidates.sort( + key=lambda item: normalize_pdf_path(item[0]) + ) + return candidates + + +def build_registered_hash_index( + conn: sqlite3.Connection, + pdf_root: Path, + papers: list[RegisteredPaper], + args: argparse.Namespace, +) -> dict[str, TextHashRecord]: + index: dict[str, TextHashRecord] = {} + updated = 0 + missing = 0 + failed = 0 + + if args.rebuild_text_hashes: + conn.execute( + """ + UPDATE paper_files + SET + file_sha256 = NULL, + text_sha256 = NULL, + text_length = NULL, + updated_at = CURRENT_TIMESTAMP + """ + ) + + for paper in papers: + pdf_path = pdf_root / paper.pdf_path + + if not pdf_path.is_file(): + missing += 1 + continue + + try: + file_sha256 = calculate_sha256(pdf_path) + cached = load_cached_text_hash( + conn, + paper.pdf_path, + ) + + if ( + cached is not None + and cached.file_sha256 == file_sha256 + ): + record = cached + else: + body, _, _ = extract_text_to_memory( + pdf_path=pdf_path, + pages=args.pages, + fallback_pages=args.fallback_pages, + min_text_length=args.min_text_length, + ) + record = TextHashRecord( + paper_id=paper.paper_id, + pdf_path=paper.pdf_path, + file_sha256=file_sha256, + text_sha256=calculate_text_sha256(body), + text_length=len(body), + ) + save_text_hash(conn, record) + updated += 1 + + if record.text_length < MIN_DUPLICATE_TEXT_LENGTH: + continue + + index.setdefault( + record.text_sha256, + record, + ) + + except Exception as exc: + failed += 1 + print( + "[警告] 既存PDFの本文ハッシュを" + f"作成できません: {paper.pdf_path}: {exc}", + file=sys.stderr, + ) + + conn.commit() + + if updated or missing or failed: + print( + "既存PDF本文ハッシュ: " + f"更新 {updated} 件, " + f"missing {missing} 件, " + f"失敗 {failed} 件" + ) + + return index + + +def find_duplicate( + body: str, + text_sha256: str, + registered_hashes: dict[str, TextHashRecord], +) -> TextHashRecord | None: + if len(body) < MIN_DUPLICATE_TEXT_LENGTH: + return None + + return registered_hashes.get(text_sha256) + + +def report_generated( + index: int, + relative_path: str, + text_path: Path, + text_length: int, + used_pages: int, + fallback_used: bool, + min_text_length: int, +) -> None: + if text_length == 0: + print( + "[警告] 文字を抽出できませんでした: " + f"{relative_path}", + file=sys.stderr, + ) + return + + if text_length < min_text_length: + print( + "[警告] 文字数が不足しています: " + f"{relative_path} ({text_length}文字)", + file=sys.stderr, + ) + return + + fallback_note = ( + f", fallback={used_pages}ページ" + if fallback_used + else "" + ) + print( + f"[{index:03d}] {relative_path} " + f"-> {text_path.name} " + f"({text_length}文字{fallback_note})" + ) + + +def process_candidate( + output_index: int, + relative_path: str, + pdf_path: Path, + output_dir: Path, + args: argparse.Namespace, + conn: sqlite3.Connection, + registered_hashes: dict[str, TextHashRecord], +) -> ProcessResult: + temp_path = output_dir / ".extracting.txt" + + body, used_pages, fallback_used = extract_pdf_text( + pdf_path=pdf_path, + output_path=temp_path, + pages=args.pages, + fallback_pages=args.fallback_pages, + min_text_length=args.min_text_length, + ) + temp_path.unlink(missing_ok=True) + + file_sha256 = calculate_sha256(pdf_path) + text_sha256 = calculate_text_sha256(body) + duplicate = find_duplicate( + body=body, + text_sha256=text_sha256, + registered_hashes=registered_hashes, + ) + + if duplicate is not None: + save_duplicate_file( + conn=conn, + paper_id=duplicate.paper_id, + pdf_path=relative_path, + file_sha256=file_sha256, + text_sha256=text_sha256, + matched_pdf_path=duplicate.pdf_path, + ) + conn.commit() + + print( + f"[重複] {relative_path}\n" + f" 既存論文ID: {duplicate.paper_id}\n" + f" 登録済PDF: {duplicate.pdf_path}\n" + " AI解析: 省略" + ) + return ProcessResult( + generated=False, + duplicate=True, + text_length=len(body), + insufficient=False, + fallback_used=fallback_used, + ) + + text_path = output_dir / f"paper-{output_index:03d}.txt" + text_length = write_text_with_metadata( + text_path=text_path, + body=body, + relative_pdf_path=relative_path, + file_sha256=file_sha256, + text_sha256=text_sha256, + file_size=pdf_path.stat().st_size, + requested_pages=args.pages, + extracted_pages=used_pages, + fallback_used=fallback_used, + min_text_length=args.min_text_length, + ) + + insufficient = text_length < args.min_text_length + + report_generated( + index=output_index, + relative_path=relative_path, + text_path=text_path, + text_length=text_length, + used_pages=used_pages, + fallback_used=fallback_used, + min_text_length=args.min_text_length, + ) + + return ProcessResult( + generated=True, + duplicate=False, + text_length=text_length, + insufficient=insufficient, + fallback_used=fallback_used, + ) + + +def clear_previous_text_files(output_dir: Path) -> None: + for text_path in output_dir.glob("paper-*.txt"): + text_path.unlink() + + +def main() -> int: + args = parse_args() + + try: + validate_args(args) + + pdf_root = args.pdf_root.expanduser().resolve() + database = args.database.expanduser().resolve() + output_dir = args.output_dir.expanduser().resolve() + + output_dir.mkdir( + parents=True, + exist_ok=True, + ) + clear_previous_text_files(output_dir) + + with sqlite3.connect(database) as conn: + conn.execute("PRAGMA foreign_keys = ON") + ensure_paper_files_table(conn) + conn.commit() + + registered_papers = load_registered_papers(conn) + known_paths = load_known_paths(conn) + registered_hashes = build_registered_hash_index( + conn=conn, + pdf_root=pdf_root, + papers=registered_papers, + args=args, + ) + + scan_dirs = ( + args.scan_dir + if args.scan_dir is not None + else list( + CONFIG.processing.scan_directories + ) + ) + scan_roots = iter_scan_roots( + pdf_root, + scan_dirs, + ) + candidates = collect_candidates( + pdf_root=pdf_root, + scan_roots=scan_roots, + known_paths=known_paths, + ) + + if not candidates: + print( + "処理対象の未登録PDFはありません。" + ) + return 10 + + generated = 0 + examined = 0 + duplicates = 0 + failed = 0 + empty = 0 + insufficient = 0 + fallback_count = 0 + + for relative_path, pdf_path in candidates: + if generated >= args.limit: + break + + examined += 1 + + try: + result = process_candidate( + output_index=generated + 1, + relative_path=relative_path, + pdf_path=pdf_path, + output_dir=output_dir, + args=args, + conn=conn, + registered_hashes=registered_hashes, + ) + + if result.duplicate: + duplicates += 1 + continue + + if result.generated: + generated += 1 + + if result.text_length == 0: + empty += 1 + + if result.insufficient: + insufficient += 1 + + if result.fallback_used: + fallback_count += 1 + + except Exception as exc: + failed += 1 + print( + f"[エラー] {relative_path}: {exc}", + file=sys.stderr, + ) + + print() + print( + f"未登録PDF候補: {len(candidates)} 件" + ) + print( + f"確認件数: {examined} 件" + ) + print( + f"重複PDF: {duplicates} 件" + ) + print( + f"AI解析対象: {generated} 件" + ) + print( + f"テキスト生成: {generated} 件" + ) + print( + f"追加ページ抽出: {fallback_count} 件" + ) + print( + f"文字数不足: {insufficient} 件" + ) + print( + f"空テキスト: {empty} 件" + ) + print( + f"失敗: {failed} 件" + ) + print( + f"出力先: {output_dir}" + ) + + if generated > 0 or duplicates > 0: + return 0 + + print( + "テキストを1件も生成できませんでした。", + file=sys.stderr, + ) + return 1 + + except Exception as exc: + print( + f"extract_text.py: {exc}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/init_db.py b/paper_db_common/scripts/init_db.py new file mode 100755 index 0000000..1420a4c --- /dev/null +++ b/paper_db_common/scripts/init_db.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sqlite3 +from pathlib import Path + +from common import CONFIG + +CATEGORIES = CONFIG.domain.categories + +SCHEMA = """ +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS categories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + description TEXT +); + +CREATE TABLE IF NOT EXISTS papers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + doi TEXT, + title TEXT NOT NULL, + title_normalized TEXT NOT NULL, + first_author TEXT, + year INTEGER, + journal TEXT, + volume TEXT, + issue TEXT, + pages TEXT, + publisher TEXT, + article_type TEXT, + language TEXT, + abstract TEXT, + summary_ja TEXT, + relevance INTEGER NOT NULL DEFAULT 0 CHECK (relevance BETWEEN 0 AND 5), + priority TEXT NOT NULL DEFAULT 'D' CHECK (priority IN ('A','B','C','D')), + review_status TEXT NOT NULL DEFAULT 'unreviewed', + category_id INTEGER, + pdf_path TEXT, + file_size INTEGER, + file_mtime TEXT, + file_sha256 TEXT, + source TEXT, + landing_url TEXT, + pdf_url TEXT, + url_source TEXT, + access_status TEXT, + url_checked_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + checked_at TEXT, + notes TEXT, + FOREIGN KEY (category_id) REFERENCES categories(id) +); + +CREATE TABLE IF NOT EXISTS authors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + name_normalized TEXT NOT NULL, + orcid TEXT, + affiliation TEXT +); + +CREATE TABLE IF NOT EXISTS paper_authors ( + paper_id INTEGER NOT NULL, + author_id INTEGER NOT NULL, + author_order INTEGER NOT NULL, + is_corresponding INTEGER NOT NULL DEFAULT 0 CHECK (is_corresponding IN (0,1)), + PRIMARY KEY (paper_id, author_order), + UNIQUE (paper_id, author_id), + FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE CASCADE, + FOREIGN KEY (author_id) REFERENCES authors(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS keywords ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + name_normalized TEXT NOT NULL UNIQUE +); + +CREATE TABLE IF NOT EXISTS paper_keywords ( + paper_id INTEGER NOT NULL, + keyword_id INTEGER NOT NULL, + PRIMARY KEY (paper_id, keyword_id), + FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE CASCADE, + FOREIGN KEY (keyword_id) REFERENCES keywords(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS processing_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pdf_path TEXT, + paper_id INTEGER, + status TEXT NOT NULL CHECK (status IN ('registered','updated','duplicate','skipped','error')), + message TEXT, + processed_at TEXT NOT NULL, + FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE SET NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_papers_doi +ON papers(doi) WHERE doi IS NOT NULL AND doi <> ''; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_papers_sha256 +ON papers(file_sha256) WHERE file_sha256 IS NOT NULL AND file_sha256 <> ''; + +CREATE INDEX IF NOT EXISTS idx_papers_title_normalized ON papers(title_normalized); +CREATE INDEX IF NOT EXISTS idx_papers_year ON papers(year); +CREATE INDEX IF NOT EXISTS idx_papers_relevance ON papers(relevance); +CREATE INDEX IF NOT EXISTS idx_papers_priority ON papers(priority); +CREATE INDEX IF NOT EXISTS idx_papers_category_id ON papers(category_id); +CREATE INDEX IF NOT EXISTS idx_authors_name_normalized ON authors(name_normalized); +CREATE INDEX IF NOT EXISTS idx_keywords_name_normalized ON keywords(name_normalized); +""" + +URL_COLUMNS = { + "landing_url": "TEXT", + "pdf_url": "TEXT", + "url_source": "TEXT", + "access_status": "TEXT", + "url_checked_at": "TEXT", +} + + +def ensure_url_columns(con: sqlite3.Connection) -> None: + columns = { + row[1] for row in con.execute("PRAGMA table_info(papers)") + } + for name, column_type in URL_COLUMNS.items(): + if name not in columns: + con.execute( + f'ALTER TABLE papers ADD COLUMN "{name}" {column_type}' + ) + + +def initialize(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(path) as con: + con.executescript(SCHEMA) + ensure_url_columns(con) + con.executemany( + "INSERT OR IGNORE INTO categories(name, description) VALUES (?, NULL)", + [(name,) for name in CATEGORIES], + ) + result = con.execute("PRAGMA integrity_check").fetchone()[0] + if result != "ok": + raise RuntimeError(result) + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument( + "database", nargs="?", default=str(CONFIG.path.database) + ) + args = p.parse_args() + initialize(Path(args.database)) + print(f"Initialized: {args.database}") + print("integrity_check: ok") + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/read_config.py b/paper_db_common/scripts/read_config.py new file mode 100755 index 0000000..acbc9c8 --- /dev/null +++ b/paper_db_common/scripts/read_config.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""config.py の設定値を表示する互換ユーティリティ。""" + +from __future__ import annotations + +import argparse +import json + +from common import CONFIG + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + data = { + "PDF_ROOT": str(CONFIG.path.pdf_root), + "SCAN_DIRECTORIES": list( + CONFIG.processing.scan_directories + ), + "REMOTE": CONFIG.remote.host, + "REMOTE_DB": CONFIG.remote.database_dir, + "REMOTE_PDF": CONFIG.remote.pdf_dir, + "MAX_PAPERS": CONFIG.processing.max_papers, + "AI_BACKEND": CONFIG.ai.backend, + } + if args.json: + print(json.dumps(data, ensure_ascii=False, indent=2)) + else: + for key, value in data.items(): + print(f"{key}={value}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/run_ai.py b/paper_db_common/scripts/run_ai.py new file mode 100755 index 0000000..e7f434d --- /dev/null +++ b/paper_db_common/scripts/run_ai.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""設定に応じてClaude CodeまたはCodex CLIを実行する。""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +from common import CONFIG + + +COMMON_ROOT = Path(__file__).resolve().parents[1] +INSTRUCTIONS_FILE = COMMON_ROOT / "CLAUDE.md" + +AI_PROMPT = """ +以下の共通指示に従い、TASK=process_new_papersを +追加確認なしで最後まで実行してください。 + +--- 共通指示開始 --- +{common_instructions} +--- 共通指示終了 --- + +tmp/text/には、未登録PDFから抽出したテキストがあります。 + +対象分野: +{domain_name} + +対象分野の説明: +{domain_description} + +categoryは次のいずれかを使用してください。 +{categories} + +日本語要約は{summary_length_ja}を目安とし、 +次の観点を優先してください。 +{summary_focus} + +relevanceは次の基準で判定してください。 +{relevance_criteria} + +Priorityは次の対応にしてください。 +- A: relevance=5 +- B: relevance=4 +- C: relevance=3 +- D: relevance=0〜2 + +行う処理は次だけです。 + +- tmp/text/にあるテキストファイルを解析する +- 各テキスト先頭のPDF_PATHをJSONのpdf_pathに使う +- DOI、書誌情報、要約、キーワード、分類を抽出する +- DOIがある場合はlanding_urlを生成する +- 必要最小限のWeb検索で公開PDFを確認する +- 各論文についてtmp/paper-*.jsonを作成する +- 作成件数と結果を日本語で報告する + +原則としてPDFは直接読まないでください。 + +次の場合のみPDF_PATHに記載された元PDFを参照できます。 + +- PDF_TEXT_EMPTYがyes +- TEXT_INSUFFICIENTがyes +- TEXT_LENGTHが{min_text_length}未満 +- テキストが文字化けしている +- タイトル、著者、発行年、要約を確認できない + +元PDFを参照する場合も、PDF_PATHに記載された1ファイルだけを読み、 +他のPDFやディレクトリを探索しないでください。 + +確認できない内容、DOI、pdf_urlは推測しないでください。 +papers.dbの直接更新、DB検証、Org出力は実行しないでください。 +最大{max_papers}件だけ処理してください。 +""".strip() + + +@dataclass(frozen=True) +class RuntimeOptions: + backend: str + max_papers: int + + +def positive_int(value: str) -> int: + number = int(value) + if number < 1: + raise argparse.ArgumentTypeError( + "1以上の整数を指定してください" + ) + return number + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Claude CodeまたはCodex CLIを実行する" + ) + parser.add_argument( + "--backend", + choices=("claude", "codex"), + default=None, + ) + parser.add_argument( + "--max-papers", + type=positive_int, + default=None, + metavar="N", + ) + return parser.parse_args() + + +def resolve_options(args: argparse.Namespace) -> RuntimeOptions: + return RuntimeOptions( + backend=args.backend or CONFIG.ai.backend, + max_papers=( + args.max_papers + if args.max_papers is not None + else CONFIG.processing.max_papers + ), + ) + + +def load_common_instructions() -> str: + if not INSTRUCTIONS_FILE.is_file(): + raise FileNotFoundError( + f"共通CLAUDE.mdがありません: {INSTRUCTIONS_FILE}" + ) + return INSTRUCTIONS_FILE.read_text(encoding="utf-8").strip() + + +def build_prompt(max_papers: int) -> str: + domain = CONFIG.domain + return AI_PROMPT.format( + common_instructions=load_common_instructions(), + domain_name=domain.name, + domain_description=domain.description, + categories=domain.categories_prompt(), + summary_length_ja=domain.summary_length_ja, + summary_focus=domain.summary_focus_prompt(), + relevance_criteria=domain.relevance_prompt(), + min_text_length=CONFIG.processing.min_text_length, + max_papers=max_papers, + ) + + +def build_claude_command() -> list[str]: + command = [ + "claude", + "--print", + "--add-dir", + str(CONFIG.path.text_dir), + "--permission-mode", + CONFIG.ai.claude_permission_mode, + "--allowedTools", + CONFIG.ai.claude_allowed_tools_argument, + ] + for path in CONFIG.scan_paths: + command.extend(["--add-dir", str(path)]) + return command + + +def build_codex_command(max_papers: int) -> list[str]: + command = [ + "codex", + "exec", + "--cd", + str(CONFIG.path.project_root), + "--sandbox", + CONFIG.ai.codex_sandbox, + ] + if CONFIG.ai.codex_model: + command.extend(["--model", CONFIG.ai.codex_model]) + if CONFIG.ai.codex_full_auto: + command.append("--full-auto") + command.append(build_prompt(max_papers)) + return command + + +def build_command(options: RuntimeOptions) -> list[str]: + if options.backend == "claude": + return build_claude_command() + if options.backend == "codex": + return build_codex_command(options.max_papers) + raise ValueError(f"未対応のAIバックエンドです: {options.backend}") + + +def require_backend_command(backend: str) -> None: + executable = {"claude": "claude", "codex": "codex"}.get(backend) + if executable is None: + raise ValueError(f"未対応のAIバックエンドです: {backend}") + if shutil.which(executable) is None: + raise RuntimeError( + f"必要なコマンドが見つかりません: {executable}" + ) + + +def main() -> int: + args = parse_args() + options = resolve_options(args) + try: + CONFIG.validate() + require_backend_command(options.backend) + command = build_command(options) + prompt = build_prompt(options.max_papers) + completed = subprocess.run( + command, + cwd=CONFIG.path.project_root, + input=(prompt if options.backend == "claude" else None), + text=True, + check=False, + ) + return completed.returncode + except Exception as exc: + print(f"run_ai.py: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/update_all.py b/paper_db_common/scripts/update_all.py new file mode 100755 index 0000000..02d714e --- /dev/null +++ b/paper_db_common/scripts/update_all.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +"""tmp/paper-*.jsonをpapers.dbへ一括登録する。""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sqlite3 +import sys +from datetime import datetime +from pathlib import Path +from typing import Any + +from common import ( + ALLOWED_CATEGORIES, + ALLOWED_PRIORITIES, + CONFIG, + normalize_doi, + normalize_text, + now_iso, + resolve_pdf_path, + sha256_file, +) + + +def parse_args() -> argparse.Namespace: + """コマンドライン引数を解析する。""" + parser = argparse.ArgumentParser( + description="tmp/paper-*.jsonをpapers.dbへ一括登録する" + ) + parser.add_argument( + "--input-dir", + default=str(CONFIG.path.tmp_dir), + ) + parser.add_argument( + "--database", + default=str(CONFIG.path.database), + ) + parser.add_argument( + "--pdf-root", + default=str(CONFIG.path.pdf_root), + ) + parser.add_argument( + "--pattern", + default="paper-*.json", + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + ) + return parser.parse_args() + + +def backup_database(database: Path) -> Path: + """DBを1回だけバックアップする。""" + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + backup = database.with_name( + f"{database.name}.bak-{stamp}" + ) + shutil.copy2(database, backup) + return backup + + +def load_json(path: Path) -> dict[str, Any]: + """論文JSONを読み込む。""" + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("JSON root must be an object") + return data + + +def validate_json(data: dict[str, Any]) -> None: + """JSONの必須項目と型を検証する。""" + required = ("title", "authors", "year", "journal", "pdf_path") + for key in required: + if key not in data: + raise ValueError(f"missing required field: {key}") + + authors = data["authors"] + if not isinstance(authors, list) or not authors: + raise ValueError("authors must be a non-empty list") + + relevance = int(data.get("relevance", 0)) + if relevance not in range(6): + raise ValueError("relevance must be 0..5") + + priority = data.get("priority", "D") + if priority not in ALLOWED_PRIORITIES: + raise ValueError("priority must be A..D") + + category = data.get("category", "Other") + if category not in ALLOWED_CATEGORIES: + raise ValueError("unknown category") + + +def add_file_metadata( + data: dict[str, Any], + pdf_root: Path, +) -> None: + """ローカルPDFの情報をJSONへ追加する。""" + actual_pdf = resolve_pdf_path( + pdf_root, + str(data["pdf_path"]), + ) + if not actual_pdf.is_file(): + raise FileNotFoundError(actual_pdf) + + stat = actual_pdf.stat() + data["file_size"] = stat.st_size + data["file_mtime"] = datetime.fromtimestamp( + stat.st_mtime + ).astimezone().isoformat(timespec="seconds") + data["file_sha256"] = sha256_file(actual_pdf) + + +def get_category_id( + connection: sqlite3.Connection, + name: str, +) -> int: + """カテゴリIDを取得する。""" + connection.execute( + "INSERT OR IGNORE INTO categories(name) VALUES (?)", + (name,), + ) + row = connection.execute( + "SELECT id FROM categories WHERE name = ?", + (name,), + ).fetchone() + return int(row[0]) + + +def get_author_id( + connection: sqlite3.Connection, + author: dict[str, Any], +) -> int: + """著者を取得または登録する。""" + name = str(author.get("name", "")).strip() + normalized = normalize_text(name) + orcid = author.get("orcid") + + row = connection.execute( + """ + SELECT id + FROM authors + WHERE name_normalized = ? + AND ((orcid = ?) OR (orcid IS NULL AND ? IS NULL)) + ORDER BY id + LIMIT 1 + """, + (normalized, orcid, orcid), + ).fetchone() + + if row: + connection.execute( + """ + UPDATE authors + SET affiliation = COALESCE(affiliation, ?), + orcid = COALESCE(orcid, ?) + WHERE id = ? + """, + (author.get("affiliation"), orcid, row[0]), + ) + return int(row[0]) + + cursor = connection.execute( + """ + INSERT INTO authors( + name, name_normalized, orcid, affiliation + ) VALUES (?, ?, ?, ?) + """, + (name, normalized, orcid, author.get("affiliation")), + ) + return int(cursor.lastrowid) + + +def get_keyword_id( + connection: sqlite3.Connection, + keyword: Any, +) -> int: + """キーワードIDを取得する。""" + name = " ".join(str(keyword).split()) + normalized = normalize_text(name) + connection.execute( + """ + INSERT OR IGNORE INTO keywords(name, name_normalized) + VALUES (?, ?) + """, + (name, normalized), + ) + row = connection.execute( + "SELECT id FROM keywords WHERE name_normalized = ?", + (normalized,), + ).fetchone() + return int(row[0]) + + +def find_existing( + connection: sqlite3.Connection, + data: dict[str, Any], +) -> sqlite3.Row | None: + """既存論文をSHA256、DOI、書誌情報の順で探す。""" + sha256 = data.get("file_sha256") + if sha256: + row = connection.execute( + "SELECT * FROM papers WHERE file_sha256 = ?", + (sha256,), + ).fetchone() + if row: + return row + + doi = normalize_doi(data.get("doi")) + if doi: + row = connection.execute( + "SELECT * FROM papers WHERE doi = ?", + (doi,), + ).fetchone() + if row: + return row + + first_author = ( + data.get("first_author") + or data["authors"][0].get("name", "") + ) + return connection.execute( + """ + SELECT * + FROM papers + WHERE title_normalized = ? + AND COALESCE(first_author, '') = ? + AND year IS ? + """, + ( + normalize_text(str(data["title"])), + first_author, + data.get("year"), + ), + ).fetchone() + + +def update_existing_paper( + connection: sqlite3.Connection, + paper_id: int, + category_id: int, + data: dict[str, Any], + now: str, +) -> None: + """既存論文の空欄を補完する。""" + connection.execute( + """ + UPDATE papers SET + doi = COALESCE(doi, ?), + journal = COALESCE(journal, ?), + volume = COALESCE(volume, ?), + issue = COALESCE(issue, ?), + pages = COALESCE(pages, ?), + publisher = COALESCE(publisher, ?), + article_type = COALESCE(article_type, ?), + language = COALESCE(language, ?), + abstract = COALESCE(abstract, ?), + summary_ja = COALESCE(summary_ja, ?), + category_id = COALESCE(category_id, ?), + pdf_path = COALESCE(pdf_path, ?), + file_size = COALESCE(file_size, ?), + file_mtime = COALESCE(file_mtime, ?), + file_sha256 = COALESCE(file_sha256, ?), + source = COALESCE(source, ?), + landing_url = COALESCE(landing_url, ?), + pdf_url = COALESCE(pdf_url, ?), + url_source = COALESCE(url_source, ?), + access_status = COALESCE(access_status, ?), + url_checked_at = COALESCE(url_checked_at, ?), + checked_at = ?, + updated_at = ? + WHERE id = ? + """, + ( + normalize_doi(data.get("doi")), + data.get("journal"), + data.get("volume"), + data.get("issue"), + data.get("pages"), + data.get("publisher"), + data.get("article_type"), + data.get("language"), + data.get("abstract"), + data.get("summary_ja"), + category_id, + data.get("pdf_path"), + data["file_size"], + data["file_mtime"], + data["file_sha256"], + data.get("source"), + data.get("landing_url"), + data.get("pdf_url"), + data.get("url_source"), + data.get("access_status"), + data.get("url_checked_at"), + now, + now, + paper_id, + ), + ) + + +def insert_paper( + connection: sqlite3.Connection, + category_id: int, + data: dict[str, Any], + now: str, +) -> int: + """新規論文を登録する。""" + first_author = ( + data.get("first_author") + or data["authors"][0].get("name", "") + ) + cursor = connection.execute( + """ + INSERT INTO papers( + doi, title, title_normalized, first_author, year, journal, + volume, issue, pages, publisher, article_type, language, + abstract, summary_ja, relevance, priority, review_status, + category_id, pdf_path, file_size, file_mtime, file_sha256, + source, landing_url, pdf_url, url_source, access_status, + url_checked_at, created_at, updated_at, checked_at, notes + ) VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + """, + ( + normalize_doi(data.get("doi")), + data["title"], + normalize_text(str(data["title"])), + first_author, + data.get("year"), + data.get("journal"), + data.get("volume"), + data.get("issue"), + data.get("pages"), + data.get("publisher"), + data.get("article_type"), + data.get("language"), + data.get("abstract"), + data.get("summary_ja"), + int(data.get("relevance", 0)), + data.get("priority", "D"), + data.get("review_status", "unreviewed"), + category_id, + data.get("pdf_path"), + data["file_size"], + data["file_mtime"], + data["file_sha256"], + data.get("source"), + data.get("landing_url"), + data.get("pdf_url"), + data.get("url_source"), + data.get("access_status"), + data.get("url_checked_at"), + now, + now, + now, + data.get("notes"), + ), + ) + return int(cursor.lastrowid) + + +def register_authors( + connection: sqlite3.Connection, + paper_id: int, + authors: list[dict[str, Any]], +) -> None: + """論文著者を登録する。""" + existing = connection.execute( + "SELECT 1 FROM paper_authors WHERE paper_id = ? LIMIT 1", + (paper_id,), + ).fetchone() + if existing: + return + + for order, author in enumerate(authors, start=1): + author_id = get_author_id(connection, author) + connection.execute( + """ + INSERT OR IGNORE INTO paper_authors( + paper_id, author_id, author_order, is_corresponding + ) VALUES (?, ?, ?, ?) + """, + ( + paper_id, + author_id, + order, + int(bool(author.get("is_corresponding", False))), + ), + ) + + +def register_keywords( + connection: sqlite3.Connection, + paper_id: int, + keywords: list[Any], +) -> None: + """論文キーワードを登録する。""" + for keyword in keywords: + keyword_id = get_keyword_id(connection, keyword) + connection.execute( + """ + INSERT OR IGNORE INTO paper_keywords( + paper_id, keyword_id + ) VALUES (?, ?) + """, + (paper_id, keyword_id), + ) + + +def register_one( + connection: sqlite3.Connection, + json_path: Path, + pdf_root: Path, +) -> dict[str, Any]: + """1件のJSONを登録する。""" + data = load_json(json_path) + validate_json(data) + add_file_metadata(data, pdf_root) + + connection.execute("BEGIN IMMEDIATE") + try: + existing = find_existing(connection, data) + category_id = get_category_id( + connection, + str(data.get("category", "Other")), + ) + now = now_iso() + + if existing: + paper_id = int(existing["id"]) + update_existing_paper( + connection, + paper_id, + category_id, + data, + now, + ) + status = "updated" + else: + paper_id = insert_paper( + connection, + category_id, + data, + now, + ) + status = "registered" + + register_authors(connection, paper_id, data["authors"]) + register_keywords( + connection, + paper_id, + data.get("keywords", []), + ) + connection.execute( + """ + INSERT INTO processing_log( + pdf_path, paper_id, status, message, processed_at + ) VALUES (?, ?, ?, ?, ?) + """, + (data["pdf_path"], paper_id, status, status, now), + ) + + integrity = connection.execute( + "PRAGMA integrity_check" + ).fetchone()[0] + if integrity != "ok": + raise RuntimeError(integrity) + + connection.commit() + return { + "status": status, + "paper_id": paper_id, + "integrity_check": integrity, + } + except Exception: + connection.rollback() + raise + + +def process_files(args: argparse.Namespace) -> int: + """対象JSONを一括処理する。""" + input_dir = Path(args.input_dir) + database = Path(args.database) + pdf_root = Path(args.pdf_root).expanduser().resolve() + files = sorted(input_dir.glob(args.pattern)) + + if not files: + print("新規JSONはありません。") + return 0 + if not database.is_file(): + raise FileNotFoundError( + f"database not found: {database}" + ) + + if CONFIG.processing.backup_database: + backup = backup_database(database) + print(f"Backup: {backup}") + + counts = {"registered": 0, "updated": 0, "failed": 0} + with sqlite3.connect(database) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys=ON") + + for path in files: + print(f"=== {path} ===") + try: + result = register_one(connection, path, pdf_root) + counts[result["status"]] += 1 + print(json.dumps(result, ensure_ascii=False)) + except Exception as exc: + counts["failed"] += 1 + print(f"ERROR: {exc}", file=sys.stderr) + if not args.continue_on_error: + return 1 + + print() + print("=== 一括登録結果 ===") + print(f"対象: {len(files)}") + print(f"新規登録: {counts['registered']}") + print(f"既存更新: {counts['updated']}") + print(f"失敗: {counts['failed']}") + return 1 if counts["failed"] else 0 + + +def main() -> int: + """メイン処理。""" + try: + return process_files(parse_args()) + except Exception as exc: + print(f"update_all.py: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/update_pdf_paths.py b/paper_db_common/scripts/update_pdf_paths.py new file mode 100755 index 0000000..18a7f21 --- /dev/null +++ b/paper_db_common/scripts/update_pdf_paths.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""SHA256が同じPDFの移動・改名をpapers.dbへ反映する。 + +PDF内容が変わってSHA256も変わった場合は更新しない。 +その場合はdelete_paper.pyでDBレコードを削除し、 +通常の登録フローで再登録する。 + +既定はdry-run。実際に更新する場合は--applyを指定する。 +""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +import sqlite3 +import sys +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from common import CONFIG, resolve_pdf_path + + +@dataclass(frozen=True) +class PdfFile: + relative_path: str + sha256: str + file_size: int + file_mtime: float + + +@dataclass(frozen=True) +class PathUpdate: + paper_id: int + title: str + old_path: str + new_path: str + sha256: str + file_size: int + file_mtime: float + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="SHA256が同じPDFの移動・改名をDBへ反映する" + ) + parser.add_argument( + "--database", + type=Path, + default=CONFIG.path.database, + ) + parser.add_argument( + "--pdf-root", + type=Path, + default=CONFIG.path.pdf_root, + ) + parser.add_argument( + "--scan-all", + action="store_true", + help="PDF_ROOT全体を走査する", + ) + parser.add_argument( + "--apply", + action="store_true", + help="更新を実際にDBへ反映する", + ) + parser.add_argument( + "--no-backup", + action="store_true", + help="更新前バックアップを作成しない", + ) + return parser.parse_args() + + +def validate_paths(database: Path, pdf_root: Path) -> None: + if not database.is_file(): + raise FileNotFoundError(database) + + if not pdf_root.is_dir(): + raise NotADirectoryError(pdf_root) + + +def scan_roots( + pdf_root: Path, + scan_all: bool, +) -> tuple[Path, ...]: + if scan_all: + return (pdf_root,) + + roots = tuple( + pdf_root / name + for name in CONFIG.processing.scan_directories + ) + + for root in roots: + if not root.is_dir(): + raise NotADirectoryError(root) + + return roots + + +def iter_pdf_paths( + roots: Iterable[Path], +) -> Iterable[Path]: + seen: set[Path] = set() + + for root in roots: + for path in root.rglob("*"): + if not path.is_file(): + continue + + if path.suffix.lower() != ".pdf": + continue + + resolved = path.resolve() + + if resolved in seen: + continue + + seen.add(resolved) + yield resolved + + +def calculate_sha256(path: Path) -> str: + digest = hashlib.sha256() + + with path.open("rb") as file: + for chunk in iter( + lambda: file.read(1024 * 1024), + b"", + ): + digest.update(chunk) + + return digest.hexdigest() + + +def inspect_pdf( + path: Path, + pdf_root: Path, +) -> PdfFile: + stat = path.stat() + + return PdfFile( + relative_path=path.relative_to( + pdf_root + ).as_posix(), + sha256=calculate_sha256(path), + file_size=stat.st_size, + file_mtime=stat.st_mtime, + ) + + +def build_sha_index( + pdf_root: Path, + roots: tuple[Path, ...], +) -> dict[str, list[PdfFile]]: + index: dict[str, list[PdfFile]] = defaultdict(list) + + paths = sorted( + iter_pdf_paths(roots), + key=lambda item: str(item).casefold(), + ) + + for number, path in enumerate(paths, start=1): + pdf = inspect_pdf(path, pdf_root) + index[pdf.sha256].append(pdf) + + print( + f"\rPDFを走査中: {number}件", + end="", + flush=True, + ) + + if paths: + print() + + return dict(index) + + +def open_database( + database: Path, +) -> sqlite3.Connection: + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys=ON") + return connection + + +def load_papers( + connection: sqlite3.Connection, +) -> list[sqlite3.Row]: + sql = """ + SELECT id, title, pdf_path, file_sha256 + FROM papers + WHERE file_sha256 IS NOT NULL + AND TRIM(file_sha256) <> '' + ORDER BY id + """ + return connection.execute(sql).fetchall() + + +def current_path_is_valid( + pdf_root: Path, + row: sqlite3.Row, +) -> bool: + if not row["pdf_path"]: + return False + + try: + path = resolve_pdf_path( + pdf_root, + row["pdf_path"], + ) + except Exception: + return False + + if not path.is_file(): + return False + + return ( + calculate_sha256(path) + == row["file_sha256"].strip().lower() + ) + + +def find_updates( + rows: Iterable[sqlite3.Row], + index: dict[str, list[PdfFile]], + pdf_root: Path, +) -> tuple[ + list[PathUpdate], + list[sqlite3.Row], + list[sqlite3.Row], +]: + updates: list[PathUpdate] = [] + ambiguous: list[sqlite3.Row] = [] + not_found: list[sqlite3.Row] = [] + + for row in rows: + if current_path_is_valid(pdf_root, row): + continue + + sha256 = row["file_sha256"].strip().lower() + candidates = index.get(sha256, []) + + if not candidates: + not_found.append(row) + continue + + if len(candidates) != 1: + ambiguous.append(row) + continue + + candidate = candidates[0] + + updates.append( + PathUpdate( + paper_id=row["id"], + title=row["title"], + old_path=row["pdf_path"] or "", + new_path=candidate.relative_path, + sha256=sha256, + file_size=candidate.file_size, + file_mtime=candidate.file_mtime, + ) + ) + + return updates, ambiguous, not_found + + +def print_updates( + updates: Iterable[PathUpdate], +) -> None: + items = list(updates) + + if not items: + return + + print() + print("=== 更新候補 ===") + + for item in items: + print(f"id: {item.paper_id}") + print(f"title: {item.title}") + print(f"old: {item.old_path or '(empty)'}") + print(f"new: {item.new_path}") + print() + + +def print_rows( + title: str, + rows: Iterable[sqlite3.Row], +) -> None: + items = list(rows) + + if not items: + return + + print() + print(f"=== {title} ===") + + for row in items: + print( + f"id={row['id']}: " + f"{row['pdf_path']} " + f"({row['title']})" + ) + + +def create_backup(database: Path) -> Path: + backup = database.with_name( + database.name + ".before-path-update" + ) + + counter = 1 + + while backup.exists(): + backup = database.with_name( + database.name + + f".before-path-update.{counter}" + ) + counter += 1 + + shutil.copy2(database, backup) + return backup + + +def apply_updates( + connection: sqlite3.Connection, + updates: Iterable[PathUpdate], +) -> int: + sql = """ + UPDATE papers + SET pdf_path = ?, + file_size = ?, + file_mtime = ? + WHERE id = ? + AND lower(file_sha256) = ? + """ + + count = 0 + + with connection: + for item in updates: + cursor = connection.execute( + sql, + ( + item.new_path, + item.file_size, + item.file_mtime, + item.paper_id, + item.sha256, + ), + ) + count += cursor.rowcount + + return count + + +def main() -> int: + args = parse_args() + + try: + database = args.database.expanduser().resolve() + pdf_root = args.pdf_root.expanduser().resolve() + + validate_paths(database, pdf_root) + + index = build_sha_index( + pdf_root, + scan_roots(pdf_root, args.scan_all), + ) + + with open_database(database) as connection: + rows = load_papers(connection) + updates, ambiguous, not_found = ( + find_updates( + rows, + index, + pdf_root, + ) + ) + + print_updates(updates) + print_rows( + "同一SHA256の候補が複数", + ambiguous, + ) + print_rows( + "SHA256一致ファイルなし", + not_found, + ) + + if args.apply and updates: + if not args.no_backup: + backup = create_backup(database) + print(f"DBバックアップ: {backup}") + + changed = apply_updates( + connection, + updates, + ) + print(f"DB更新: {changed} 件") + + print() + print("=== 概要 ===") + print( + "走査PDF: " + f"{sum(len(v) for v in index.values())} 件" + ) + print(f"DB照合対象: {len(rows)} 件") + print(f"更新候補: {len(updates)} 件") + print(f"曖昧な一致: {len(ambiguous)} 件") + print(f"一致なし: {len(not_found)} 件") + print( + "実行モード: " + + ("更新実行" if args.apply else "dry-run") + ) + + if updates and not args.apply: + print() + print( + "更新するには --apply を指定してください。" + ) + + return 0 + + except KeyboardInterrupt: + return 130 + + except Exception as exc: + print( + f"update_pdf_paths.py: {exc}", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/scripts/validate_db.py b/paper_db_common/scripts/validate_db.py new file mode 100755 index 0000000..9008fbb --- /dev/null +++ b/paper_db_common/scripts/validate_db.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sqlite3 +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from common import CONFIG, resolve_pdf_path + + +@dataclass(frozen=True) +class MissingPdf: + """見つからないPDFの情報。""" + + paper_id: int + title: str + pdf_path: str + resolved_path: str | None + reason: str + + +@dataclass(frozen=True) +class IntegrityResult: + """SQLite整合性検証の結果。""" + + integrity: str + foreign_key_count: int + errors: tuple[str, ...] + + +def parse_args() -> argparse.Namespace: + """コマンドライン引数を解析する。""" + parser = argparse.ArgumentParser( + description="papers.dbの整合性を検証する" + ) + parser.add_argument( + "database", + nargs="?", + default=str(CONFIG.path.database), + help="検証するSQLiteデータベース", + ) + parser.add_argument( + "--pdf-root", + default=str(CONFIG.path.pdf_root), + help="PDF_ROOTのパス", + ) + return parser.parse_args() + + +def open_database(database: Path) -> sqlite3.Connection: + """SQLiteデータベースへ接続する。""" + connection = sqlite3.connect(database) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys=ON") + return connection + + +def check_integrity( + connection: sqlite3.Connection, +) -> IntegrityResult: + """SQLite本体と外部キーの整合性を検証する。""" + errors: list[str] = [] + + integrity = connection.execute( + "PRAGMA integrity_check" + ).fetchone()[0] + + if integrity != "ok": + errors.append(f"integrity_check: {integrity}") + + foreign_keys = connection.execute( + "PRAGMA foreign_key_check" + ).fetchall() + + foreign_key_count = len(foreign_keys) + + if foreign_key_count: + errors.append( + f"foreign keys: {foreign_key_count}" + ) + + return IntegrityResult( + integrity=integrity, + foreign_key_count=foreign_key_count, + errors=tuple(errors), + ) + + +def validation_queries() -> tuple[tuple[str, str], ...]: + """DB内容の検証用SQLを返す。""" + return ( + ( + "duplicate DOI", + """ + SELECT doi + FROM papers + WHERE doi IS NOT NULL + AND doi <> '' + GROUP BY doi + HAVING COUNT(*) > 1 + """, + ), + ( + "duplicate SHA256", + """ + SELECT file_sha256 + FROM papers + WHERE file_sha256 IS NOT NULL + AND file_sha256 <> '' + GROUP BY file_sha256 + HAVING COUNT(*) > 1 + """, + ), + ( + "empty title", + """ + SELECT id + FROM papers + WHERE title IS NULL + OR TRIM(title) = '' + """, + ), + ( + "invalid relevance", + """ + SELECT id + FROM papers + WHERE relevance NOT BETWEEN 0 AND 5 + """, + ), + ( + "invalid priority", + """ + SELECT id + FROM papers + WHERE priority NOT IN ('A', 'B', 'C', 'D') + """, + ), + ) + + +def check_database_values( + connection: sqlite3.Connection, +) -> list[str]: + """重複や値域を検証する。""" + errors: list[str] = [] + + for label, sql in validation_queries(): + rows = connection.execute(sql).fetchall() + + if rows: + errors.append(f"{label}: {len(rows)}") + + return errors + + +def iter_papers_with_pdf( + connection: sqlite3.Connection, +) -> Iterable[sqlite3.Row]: + """PDFパスが登録された論文を返す。""" + sql = """ + SELECT id, title, pdf_path + FROM papers + WHERE pdf_path IS NOT NULL + AND TRIM(pdf_path) <> '' + ORDER BY id + """ + return connection.execute(sql) + + +def inspect_pdf( + pdf_root: Path, + row: sqlite3.Row, +) -> MissingPdf | None: + """1件のPDFが存在するか確認する。""" + try: + path = resolve_pdf_path( + pdf_root, + row["pdf_path"], + ) + + if path.is_file(): + return None + + return MissingPdf( + paper_id=row["id"], + title=row["title"], + pdf_path=row["pdf_path"], + resolved_path=str(path), + reason="file does not exist", + ) + + except Exception as exc: + return MissingPdf( + paper_id=row["id"], + title=row["title"], + pdf_path=row["pdf_path"], + resolved_path=None, + reason=f"{type(exc).__name__}: {exc}", + ) + + +def find_missing_pdfs( + connection: sqlite3.Connection, + pdf_root: Path, +) -> list[MissingPdf]: + """存在しないPDFを検索する。""" + missing: list[MissingPdf] = [] + + for row in iter_papers_with_pdf(connection): + item = inspect_pdf(pdf_root, row) + + if item is not None: + missing.append(item) + + return missing + + +def print_missing_pdfs( + missing: Iterable[MissingPdf], +) -> None: + """見つからないPDFの詳細を表示する。""" + items = list(missing) + + if not items: + return + + print() + print("=== Missing PDFs ===") + + for item in items: + print(f"paper_id: {item.paper_id}") + print(f"title: {item.title}") + print(f"pdf_path: {item.pdf_path}") + print( + "resolved_path:", + item.resolved_path + or "(resolve failed)", + ) + print(f"reason: {item.reason}") + print() + + +def print_summary( + integrity: IntegrityResult, + errors: list[str], +) -> None: + """検証結果を表示する。""" + print(f"integrity_check: {integrity.integrity}") + print( + "foreign_key_violations:", + integrity.foreign_key_count, + ) + print(f"errors: {len(errors)}") + + for error in errors: + print(f"- {error}") + + +def validate( + database: Path, + pdf_root: Path, +) -> list[str]: + """データベース全体を検証する。""" + with open_database(database) as connection: + integrity = check_integrity(connection) + value_errors = check_database_values( + connection + ) + missing = find_missing_pdfs( + connection, + pdf_root, + ) + + errors = list(integrity.errors) + errors.extend(value_errors) + + if missing: + errors.append( + f"missing PDF: {len(missing)}" + ) + + print_missing_pdfs(missing) + print_summary(integrity, errors) + + return errors + + +def main() -> int: + """メイン処理。""" + args = parse_args() + + database = Path( + args.database + ).expanduser().resolve() + + pdf_root = Path( + args.pdf_root + ).expanduser().resolve() + + errors = validate( + database, + pdf_root, + ) + + return 1 if errors else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/paper_db_common/templates/project/README.org b/paper_db_common/templates/project/README.org new file mode 100644 index 0000000..cf8c90c --- /dev/null +++ b/paper_db_common/templates/project/README.org @@ -0,0 +1,40 @@ +#+TITLE: 論文データベースプロジェクト + +* 構成 + +このディレクトリには、分野固有の設定とデータだけを置く。 + +#+begin_example +config.py +papers.db +papers.org +papers.publish.db +logs/ +tmp/ +#+end_example + +=CLAUDE.md= とPythonスクリプトは共通側に置かれるため、 +このディレクトリには作成しない。 + +* 初期設定 + +=config.py= の次の項目を編集する。 + +- =pdf_root= +- =scan_directories= +- =domain= +- 必要な場合だけ =remote= + +* 実行 + +プロジェクトディレクトリへ移動して実行する。 + +#+begin_src sh +/path/to/paper_db_common/publish.py +#+end_src + +公開用DBとPDFも転送する場合は次を使う。 + +#+begin_src sh +/path/to/paper_db_common/publish.py --publish +#+end_src diff --git a/paper_db_common/templates/project/config.py b/paper_db_common/templates/project/config.py new file mode 100755 index 0000000..c72e87c --- /dev/null +++ b/paper_db_common/templates/project/config.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +from pathlib import Path + +from paperdb_config import AIConfig +from paperdb_config import Config +from paperdb_config import DomainConfig +from paperdb_config import PathConfig +from paperdb_config import ProcessingConfig +from paperdb_config import RemoteConfig + + +ROOT = Path(__file__).resolve().parent + +CONFIG = Config( + # pdfのパスを指定 + path=PathConfig( + project_root=ROOT, + pdf_root=Path.home() / "Documents" / "papers", + database=ROOT / "papers.db", + publish_database=ROOT / "papers.publish.db", + tmp_dir=ROOT / "tmp", + text_dir=ROOT / "tmp" / "text", + log_dir=ROOT / "logs", + papers_org=ROOT / "papers.org", + ), + # scan_directoriesにpdfを置いているdirectryをlist + processing=ProcessingConfig( + scan_directories=("user1", "user2",), + ignore_directories=("old", "tmp", "trash"), + max_papers=3, + initial_pages=2, + fallback_pages=5, + min_text_length=1500, + update_database=True, + validate_database=True, + export_org=True, + backup_database=True, + ), + # categoriesに分類をlist, 分野名にわかりやすいname + domain=DomainConfig( + name="分野名", + description="収集対象とする論文分野の説明", + categories=( + "Review", + "Other", + ), + default_category="Other", + summary_length_ja="100〜200字程度", + summary_focus=( + "研究目的", + "手法", + "主な結果", + "新規性", + "対象分野との関連", + ), + relevance_criteria=( + "対象分野と無関係", + "わずかに関連", + "周辺分野", + "関連する基礎研究", + "対象分野へ直接応用可能", + "対象分野そのもの", + ), + ), + remote=RemoteConfig( + host="user@example.org", + database_dir="/home/user/public_html/db", + pdf_dir="/home/user/public_html/pdf", + ), + ai=AIConfig( + backend="claude", + claude_permission_mode="dontAsk", + claude_allowed_tools=( + "Read", + "Glob", + "Grep", + "Write", + "Edit", + "WebSearch", + "WebFetch", + ), + codex_model=None, + codex_sandbox="workspace-write", + codex_full_auto=True, + ), +)