Initial commit
This commit is contained in:
+41
@@ -0,0 +1,41 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.so
|
||||
*.egg-info/
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
# Editors
|
||||
*~
|
||||
\#*\#
|
||||
.\#*
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# Runtime
|
||||
logs/
|
||||
tmp/
|
||||
|
||||
# Databases
|
||||
*.db
|
||||
*.db-journal
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
|
||||
# Generated
|
||||
papers.org
|
||||
summary.org
|
||||
paper-*.json
|
||||
|
||||
# Cache
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
|
||||
# Build
|
||||
build/
|
||||
dist/
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Takahiro Ohkubo
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
#+TITLE: paper_db
|
||||
|
||||
* paper_db
|
||||
|
||||
paper_db は、複数分野で共通利用できる論文データベース基盤です。
|
||||
|
||||
** 特徴
|
||||
|
||||
- PDFから本文抽出
|
||||
- Claude / Codex による書誌情報・要約生成
|
||||
- SQLiteで論文管理
|
||||
- Org-modeへのエクスポート
|
||||
- 分野ごとの差分は config.py のみ
|
||||
|
||||
** ディレクトリ構成
|
||||
|
||||
#+begin_example
|
||||
paper_db/
|
||||
├── paper_db_common/
|
||||
├── examples/
|
||||
│ └── wglass_db/
|
||||
└── README.org
|
||||
#+end_example
|
||||
|
||||
** 新しいプロジェクトの作成
|
||||
|
||||
#+begin_src sh
|
||||
paper_db_common/new_project.py ~/research/my_db
|
||||
#+end_src
|
||||
|
||||
** 論文処理
|
||||
|
||||
#+begin_src sh
|
||||
cd ~/research/my_db
|
||||
/path/to/paper_db_common/publish.py process
|
||||
#+end_src
|
||||
|
||||
** 公開
|
||||
|
||||
#+begin_src sh
|
||||
/path/to/paper_db_common/publish.py publish
|
||||
#+end_src
|
||||
|
||||
** ライセンス
|
||||
|
||||
MIT License
|
||||
@@ -0,0 +1,40 @@
|
||||
#+TITLE: 論文データベースプロジェクト
|
||||
|
||||
* 構成
|
||||
|
||||
このディレクトリには、分野固有の設定とデータだけを置く。
|
||||
|
||||
#+begin_example
|
||||
config.py
|
||||
papers.db
|
||||
papers.org
|
||||
papers.publish.db
|
||||
logs/
|
||||
tmp/
|
||||
#+end_example
|
||||
|
||||
=CLAUDE.md= とPythonスクリプトは共通側に置かれるため、
|
||||
このディレクトリには作成しない。
|
||||
|
||||
* 初期設定
|
||||
|
||||
=config.py= の次の項目を編集する。
|
||||
|
||||
- =pdf_root=
|
||||
- =scan_directories=
|
||||
- =domain=
|
||||
- 必要な場合だけ =remote=
|
||||
|
||||
* 実行
|
||||
|
||||
プロジェクトディレクトリへ移動して実行する。
|
||||
|
||||
#+begin_src sh
|
||||
/path/to/paper_db_common/publish.py
|
||||
#+end_src
|
||||
|
||||
公開用DBとPDFも転送する場合は次を使う。
|
||||
|
||||
#+begin_src sh
|
||||
/path/to/paper_db_common/publish.py --publish
|
||||
#+end_src
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from paperdb_config import AIConfig
|
||||
from paperdb_config import Config
|
||||
from paperdb_config import DomainConfig
|
||||
from paperdb_config import PathConfig
|
||||
from paperdb_config import ProcessingConfig
|
||||
from paperdb_config import RemoteConfig
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
CONFIG = Config(
|
||||
path=PathConfig(
|
||||
project_root=ROOT,
|
||||
pdf_root=(
|
||||
Path.home()
|
||||
/ "G_drive"
|
||||
/ "ガラス固化体データベース"
|
||||
),
|
||||
database=ROOT / "papers.db",
|
||||
publish_database=ROOT / "papers.publish.db",
|
||||
tmp_dir=ROOT / "tmp",
|
||||
text_dir=ROOT / "tmp" / "text",
|
||||
log_dir=ROOT / "logs",
|
||||
papers_org=ROOT / "papers.org",
|
||||
),
|
||||
remote=RemoteConfig(
|
||||
host="wglass@amorphous",
|
||||
database_dir="/home/wglass/public_html/db",
|
||||
pdf_dir="/home/wglass/public_html/pdf",
|
||||
),
|
||||
ai=AIConfig(
|
||||
backend="claude",
|
||||
claude_permission_mode="dontAsk",
|
||||
claude_allowed_tools=(
|
||||
"Read",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Write",
|
||||
"Edit",
|
||||
"WebSearch",
|
||||
"WebFetch",
|
||||
),
|
||||
codex_model=None,
|
||||
codex_sandbox="workspace-write",
|
||||
codex_full_auto=True,
|
||||
),
|
||||
processing=ProcessingConfig(
|
||||
scan_directories=(
|
||||
"千葉大_大窪",
|
||||
"JAEA_三ツ井",
|
||||
"RWMC_稲垣",
|
||||
),
|
||||
ignore_directories=("old", "tmp", "trash"),
|
||||
max_papers=1,
|
||||
initial_pages=2,
|
||||
fallback_pages=5,
|
||||
min_text_length=1500,
|
||||
update_database=True,
|
||||
validate_database=True,
|
||||
export_org=True,
|
||||
backup_database=True,
|
||||
),
|
||||
domain=DomainConfig(
|
||||
name="ガラス固化体",
|
||||
description=(
|
||||
"高レベル放射性廃棄物ガラス、廃棄物ガラス、"
|
||||
"ホウケイ酸塩ガラス、耐久性、変質、放射線影響、"
|
||||
"核種保持に関する論文"
|
||||
),
|
||||
categories=(
|
||||
"Review",
|
||||
"Glass structure",
|
||||
"Crystallization",
|
||||
"Melting",
|
||||
"Corrosion",
|
||||
"Durability",
|
||||
"Alteration layer",
|
||||
"Radiation effects",
|
||||
"Simulation",
|
||||
"Molecular dynamics",
|
||||
"Machine learning",
|
||||
"NMR",
|
||||
"Raman",
|
||||
"XAFS",
|
||||
"Diffraction",
|
||||
"Actinides",
|
||||
"Fission products",
|
||||
"Other",
|
||||
),
|
||||
default_category="Other",
|
||||
summary_length_ja="100〜200字程度",
|
||||
summary_focus=(
|
||||
"研究目的",
|
||||
"使用した実験または計算手法",
|
||||
"主な結果",
|
||||
"新規性",
|
||||
"ガラス固化体研究との関連",
|
||||
),
|
||||
relevance_criteria=(
|
||||
"無関係",
|
||||
"わずかに関連",
|
||||
"周辺分野",
|
||||
"関連するガラス科学",
|
||||
"ガラス固化体研究へ直接応用可能",
|
||||
"高レベル放射性廃棄物ガラスそのもの",
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,237 @@
|
||||
# 論文データベース共通処理
|
||||
|
||||
## 1. 役割
|
||||
|
||||
このプロジェクトはPDFファイルそのものを保持しない。
|
||||
|
||||
PDFは`config.py`の`CONFIG.path.pdf_root`で指定された外部ディレクトリに
|
||||
保存されている。
|
||||
|
||||
`papers.db`を唯一のマスターデータベースとする。
|
||||
`papers.org`は閲覧・検索・レビュー用であり、マスターデータではない。
|
||||
|
||||
分野名、カテゴリ、要約観点、relevance基準は`config.py`の
|
||||
`CONFIG.domain`を唯一の定義元とする。
|
||||
|
||||
## 2. 処理前に必ず行うこと
|
||||
|
||||
1. プロジェクトの`config.py`を読む。
|
||||
2. `CONFIG`を読み、処理条件と分野設定を確認する。
|
||||
3. 処理対象は`publish.py`が`tmp/text/`へ準備する。
|
||||
4. `tmp/text/`にない論文は処理しない。
|
||||
5. 設定不足や存在しないパスがあれば処理を開始しない。
|
||||
|
||||
## 3. TASK=process_new_papers
|
||||
|
||||
AIが行う処理は、`tmp/text/`の解析とJSON生成までとする。
|
||||
|
||||
1. `tmp/text/paper-*.txt`だけを処理対象とする。
|
||||
2. 各テキスト先頭の`PDF_PATH`をJSONの`pdf_path`に使用する。
|
||||
3. 書誌情報、要約、キーワード、分類、Webリンクを抽出する。
|
||||
4. 各論文について`tmp/paper-*.json`を生成する。
|
||||
5. 作成したJSONファイルと処理結果を報告する。
|
||||
|
||||
原則として元PDFは直接読まない。次の場合のみ、`PDF_PATH`に
|
||||
記載された1ファイルを参照してよい。
|
||||
|
||||
- `PDF_TEXT_EMPTY`が`yes`
|
||||
- `TEXT_INSUFFICIENT`が`yes`
|
||||
- テキストが文字化けしている
|
||||
- タイトル、著者、発行年、要約を確認できない
|
||||
|
||||
他のPDFやディレクトリは探索しない。
|
||||
|
||||
AIは次を実行しない。
|
||||
|
||||
- DB登録スクリプト
|
||||
- DB検証スクリプト
|
||||
- Org出力スクリプト
|
||||
- `papers.db`の直接更新
|
||||
|
||||
JSON生成後のDB登録、検証、Org出力は`publish.py`が実行する。
|
||||
|
||||
## 4. PDFパス
|
||||
|
||||
`papers.db`の`pdf_path`には、`CONFIG.path.pdf_root`からの
|
||||
相対パスだけを保存する。
|
||||
|
||||
絶対パスは保存しない。PDFは削除、移動、改名しない。
|
||||
|
||||
## 5. 抽出項目
|
||||
|
||||
必須:
|
||||
|
||||
- title
|
||||
- authors
|
||||
- first_author
|
||||
- year
|
||||
- journal
|
||||
- doi
|
||||
- pdf_path
|
||||
|
||||
可能なら:
|
||||
|
||||
- volume
|
||||
- issue
|
||||
- pages
|
||||
- publisher
|
||||
- article_type
|
||||
- language
|
||||
- abstract
|
||||
- ORCID
|
||||
- affiliation
|
||||
|
||||
AIが付与:
|
||||
|
||||
- summary_ja
|
||||
- keywords
|
||||
- category
|
||||
- relevance
|
||||
- priority
|
||||
|
||||
## 6. JSONフォーマット
|
||||
|
||||
```json
|
||||
{
|
||||
"doi": null,
|
||||
"title": "",
|
||||
"authors": [
|
||||
{
|
||||
"name": "",
|
||||
"orcid": null,
|
||||
"affiliation": null,
|
||||
"is_corresponding": false
|
||||
}
|
||||
],
|
||||
"first_author": "",
|
||||
"year": null,
|
||||
"journal": "",
|
||||
"volume": null,
|
||||
"issue": null,
|
||||
"pages": null,
|
||||
"publisher": null,
|
||||
"article_type": "journal article",
|
||||
"language": "en",
|
||||
"abstract": null,
|
||||
"summary_ja": "",
|
||||
"keywords": [],
|
||||
"category": "Other",
|
||||
"relevance": 3,
|
||||
"priority": "C",
|
||||
"pdf_path": "",
|
||||
"source": "local PDF",
|
||||
"landing_url": null,
|
||||
"pdf_url": null,
|
||||
"url_source": "unknown",
|
||||
"access_status": "unknown",
|
||||
"url_checked_at": null
|
||||
}
|
||||
```
|
||||
|
||||
### authors
|
||||
|
||||
`authors`は必ず配列とする。複数著者は論文中の順番で並べる。
|
||||
取得できない場合は、`name`を`unknown`とした要素を1件入れる。
|
||||
|
||||
### first_author
|
||||
|
||||
`authors`配列の先頭の`name`と一致させる。
|
||||
著者不明の場合は`unknown`とする。
|
||||
|
||||
### keywords
|
||||
|
||||
必ず文字列の配列とする。取得できない場合は空配列とする。
|
||||
|
||||
### category
|
||||
|
||||
`config.py`の`CONFIG.domain.categories`から、最も適切なものを
|
||||
1つ選択する。該当するカテゴリがない場合は
|
||||
`CONFIG.domain.default_category`を使用する。
|
||||
|
||||
### DOI
|
||||
|
||||
- 推測しない。
|
||||
- 小文字化する。
|
||||
- `https://doi.org/`などの接頭辞を除去する。
|
||||
- 不明な場合は`null`とする。
|
||||
|
||||
### year
|
||||
|
||||
整数または`null`とする。
|
||||
|
||||
### article_type
|
||||
|
||||
例:
|
||||
|
||||
- journal article
|
||||
- review
|
||||
- report
|
||||
- conference paper
|
||||
- book chapter
|
||||
- other
|
||||
|
||||
## 7. Web上の論文リンク
|
||||
|
||||
DOIがある場合、`landing_url`は`https://doi.org/{doi}`とする。
|
||||
PDF URLは推測しない。実際に公開されていることを確認できた場合のみ
|
||||
`pdf_url`へ保存する。
|
||||
|
||||
`url_source`の例:
|
||||
|
||||
- doi
|
||||
- crossref
|
||||
- unpaywall
|
||||
- publisher
|
||||
- institutional_repository
|
||||
- government_repository
|
||||
- manual
|
||||
- unknown
|
||||
|
||||
`access_status`:
|
||||
|
||||
- open
|
||||
- restricted
|
||||
- unknown
|
||||
|
||||
## 8. 日本語要約
|
||||
|
||||
`CONFIG.domain.summary_length_ja`を目安とする。
|
||||
`CONFIG.domain.summary_focus`に記載された観点を優先する。
|
||||
本文から確認できないことは推測しない。
|
||||
|
||||
## 9. RelevanceとPriority
|
||||
|
||||
`relevance`は0から5の整数とする。
|
||||
判断基準は`CONFIG.domain.relevance_criteria`を使用する。
|
||||
|
||||
Priorityは次の対応に固定する。
|
||||
|
||||
- A: relevance=5
|
||||
- B: relevance=4
|
||||
- C: relevance=3
|
||||
- D: relevance=0から2
|
||||
|
||||
## 10. JSONの妥当性
|
||||
|
||||
次を厳守する。
|
||||
|
||||
- `authors`: 空でない配列
|
||||
- `keywords`: 配列
|
||||
- `year`: 整数または`null`
|
||||
- `doi`: 文字列または`null`
|
||||
- `category`: `CONFIG.domain.categories`のいずれか
|
||||
- `relevance`: 0から5の整数
|
||||
- `priority`: A、B、C、Dのいずれか
|
||||
- `is_corresponding`: 真偽値
|
||||
- `pdf_path`: PDFルートからの相対パス
|
||||
|
||||
## 11. 禁止事項
|
||||
|
||||
- `papers.db`の削除
|
||||
- 場当たり的なスキーマ変更
|
||||
- PDFの削除、移動、改名
|
||||
- DOIや論文内容の推測
|
||||
- Orgをマスターとして扱う
|
||||
- 人が入力したメモの削除
|
||||
- `authors`や`keywords`を文字列として出力すること
|
||||
- `pdf_url`を推測で生成すること
|
||||
@@ -0,0 +1,75 @@
|
||||
#+TITLE: paper_db_common
|
||||
|
||||
* 概要
|
||||
|
||||
複数分野の論文データベースで共有するコード一式である。
|
||||
各DBプロジェクトは任意の場所に配置できる。
|
||||
|
||||
共通側には次を置く。
|
||||
|
||||
#+begin_example
|
||||
paper_db_common/
|
||||
├── CLAUDE.md
|
||||
├── new_project.py
|
||||
├── paperdb_config.py
|
||||
├── project_loader.py
|
||||
├── publish.py
|
||||
├── scripts/
|
||||
└── templates/
|
||||
#+end_example
|
||||
|
||||
各DB側には分野固有の設定とデータだけを置く。
|
||||
|
||||
#+begin_example
|
||||
wglass_db/
|
||||
├── config.py
|
||||
├── papers.db
|
||||
├── papers.org
|
||||
├── papers.publish.db
|
||||
├── logs/
|
||||
└── tmp/
|
||||
#+end_example
|
||||
|
||||
各DB側に =CLAUDE.md= は不要である。
|
||||
=run_ai.py= が共通側の =CLAUDE.md= を読み込み、
|
||||
分野固有の条件は =config.py= からプロンプトへ追加する。
|
||||
|
||||
* 新規DBの作成
|
||||
|
||||
#+begin_src sh
|
||||
/path/to/paper_db_common/new_project.py /path/to/new_db
|
||||
#+end_src
|
||||
|
||||
既存の空でないディレクトリへ作成する場合だけ
|
||||
=--force= を指定する。
|
||||
|
||||
* 実行モード
|
||||
|
||||
=publish.py= は実行モードの指定を必須とする。
|
||||
|
||||
- =process= :: PDF抽出、AI解析、JSON登録によるDB更新
|
||||
- =sync= :: PDFパス更新、DB検証、papers.org出力
|
||||
- =publish= :: PDF同期、公開用DB作成、リモート反映
|
||||
- =all= :: process、sync、publishを順番に実行
|
||||
|
||||
#+begin_src sh
|
||||
cd /path/to/new_db
|
||||
/path/to/paper_db_common/publish.py process
|
||||
/path/to/paper_db_common/publish.py sync
|
||||
/path/to/paper_db_common/publish.py publish
|
||||
/path/to/paper_db_common/publish.py all
|
||||
#+end_src
|
||||
|
||||
別の場所から実行する場合は =--config= を指定する。
|
||||
|
||||
#+begin_src sh
|
||||
/path/to/paper_db_common/publish.py process \
|
||||
--config /path/to/new_db/config.py
|
||||
#+end_src
|
||||
|
||||
* パスの扱い
|
||||
|
||||
各DBの =config.py= に共通コードの絶対パスや
|
||||
=COMMON_ROOT=、=sys.path= 操作を書く必要はない。
|
||||
共通側は =__file__= から自身の位置を決定する。
|
||||
=config.py= のディレクトリが =project_root= となる。
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""共通テンプレートから新しい論文DBプロジェクトを作成する。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
COMMON_ROOT = Path(__file__).resolve().parent
|
||||
TEMPLATE_DIR = COMMON_ROOT / "templates" / "project"
|
||||
TEMPLATE_FILES = (
|
||||
"config.py",
|
||||
"README.org",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="新しい分野DBを作成する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"directory",
|
||||
type=Path,
|
||||
help="作成するプロジェクトディレクトリ",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="既存ファイルを上書きする",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_destination(
|
||||
destination: Path,
|
||||
force: bool,
|
||||
) -> None:
|
||||
if not destination.exists():
|
||||
return
|
||||
if not destination.is_dir():
|
||||
raise NotADirectoryError(destination)
|
||||
if any(destination.iterdir()) and not force:
|
||||
raise FileExistsError(
|
||||
f"空でないディレクトリです: {destination}"
|
||||
)
|
||||
|
||||
|
||||
def copy_template_files(destination: Path) -> None:
|
||||
for name in TEMPLATE_FILES:
|
||||
source = TEMPLATE_DIR / name
|
||||
if not source.is_file():
|
||||
raise FileNotFoundError(source)
|
||||
shutil.copy2(source, destination / name)
|
||||
|
||||
|
||||
def create_directories(destination: Path) -> None:
|
||||
(destination / "tmp" / "text").mkdir(
|
||||
parents=True,
|
||||
exist_ok=True,
|
||||
)
|
||||
(destination / "logs").mkdir(
|
||||
parents=True,
|
||||
exist_ok=True,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
destination = args.directory.expanduser().resolve()
|
||||
validate_destination(destination, args.force)
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
copy_template_files(destination)
|
||||
create_directories(destination)
|
||||
print(destination)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PathConfig:
|
||||
project_root: Path
|
||||
pdf_root: Path
|
||||
database: Path
|
||||
publish_database: Path
|
||||
tmp_dir: Path
|
||||
text_dir: Path
|
||||
log_dir: Path
|
||||
papers_org: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RemoteConfig:
|
||||
host: str
|
||||
database_dir: str
|
||||
pdf_dir: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AIConfig:
|
||||
backend: str
|
||||
claude_permission_mode: str
|
||||
claude_allowed_tools: tuple[str, ...]
|
||||
codex_model: str | None
|
||||
codex_sandbox: str
|
||||
codex_full_auto: bool
|
||||
|
||||
@property
|
||||
def claude_allowed_tools_argument(self) -> str:
|
||||
return ",".join(self.claude_allowed_tools)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProcessingConfig:
|
||||
scan_directories: tuple[str, ...]
|
||||
ignore_directories: tuple[str, ...]
|
||||
max_papers: int
|
||||
initial_pages: int
|
||||
fallback_pages: int
|
||||
min_text_length: int
|
||||
update_database: bool
|
||||
validate_database: bool
|
||||
export_org: bool
|
||||
backup_database: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DomainConfig:
|
||||
name: str
|
||||
description: str
|
||||
categories: tuple[str, ...]
|
||||
default_category: str
|
||||
summary_length_ja: str
|
||||
summary_focus: tuple[str, ...]
|
||||
relevance_criteria: tuple[str, ...]
|
||||
priorities: tuple[str, ...] = ("A", "B", "C", "D")
|
||||
|
||||
def validate(self) -> None:
|
||||
if not self.name.strip():
|
||||
raise ValueError("domain.nameが空です")
|
||||
if not self.description.strip():
|
||||
raise ValueError("domain.descriptionが空です")
|
||||
if not self.categories:
|
||||
raise ValueError("domain.categoriesが空です")
|
||||
if len(set(self.categories)) != len(self.categories):
|
||||
raise ValueError("domain.categoriesに重複があります")
|
||||
if self.default_category not in self.categories:
|
||||
raise ValueError(
|
||||
"default_categoryをcategoriesに含めてください"
|
||||
)
|
||||
if len(self.relevance_criteria) != 6:
|
||||
raise ValueError(
|
||||
"relevance_criteriaは0から5の6項目にしてください"
|
||||
)
|
||||
|
||||
def categories_prompt(self) -> str:
|
||||
return "\n".join(
|
||||
f"- {category}" for category in self.categories
|
||||
)
|
||||
|
||||
def summary_focus_prompt(self) -> str:
|
||||
return "\n".join(
|
||||
f"- {item}" for item in self.summary_focus
|
||||
)
|
||||
|
||||
def relevance_prompt(self) -> str:
|
||||
return "\n".join(
|
||||
f"- {score}: {description}"
|
||||
for score, description in enumerate(
|
||||
self.relevance_criteria
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
path: PathConfig
|
||||
remote: RemoteConfig
|
||||
ai: AIConfig
|
||||
processing: ProcessingConfig
|
||||
domain: DomainConfig
|
||||
|
||||
@property
|
||||
def scan_paths(self) -> tuple[Path, ...]:
|
||||
return tuple(
|
||||
self.path.pdf_root / directory
|
||||
for directory in self.processing.scan_directories
|
||||
)
|
||||
|
||||
def validate(self, *, require_remote: bool = False) -> None:
|
||||
root = self.path.project_root.resolve()
|
||||
if not root.is_dir():
|
||||
raise NotADirectoryError(root)
|
||||
if not self.path.pdf_root.is_dir():
|
||||
raise NotADirectoryError(
|
||||
f"PDF_ROOTがありません: {self.path.pdf_root}"
|
||||
)
|
||||
if not self.processing.scan_directories:
|
||||
raise ValueError("scan_directoriesが空です")
|
||||
for path in self.scan_paths:
|
||||
if not path.is_dir():
|
||||
raise NotADirectoryError(
|
||||
f"探索ディレクトリがありません: {path}"
|
||||
)
|
||||
if self.processing.max_papers < 1:
|
||||
raise ValueError("max_papersは1以上にしてください")
|
||||
if self.processing.initial_pages < 1:
|
||||
raise ValueError("initial_pagesは1以上にしてください")
|
||||
if (
|
||||
self.processing.fallback_pages
|
||||
< self.processing.initial_pages
|
||||
):
|
||||
raise ValueError(
|
||||
"fallback_pagesはinitial_pages以上にしてください"
|
||||
)
|
||||
if self.processing.min_text_length < 0:
|
||||
raise ValueError(
|
||||
"min_text_lengthは0以上にしてください"
|
||||
)
|
||||
if self.ai.backend.lower() not in {"claude", "codex"}:
|
||||
raise ValueError(
|
||||
"ai.backendはclaudeまたはcodexにしてください"
|
||||
)
|
||||
self.domain.validate()
|
||||
if require_remote:
|
||||
values = (
|
||||
self.remote.host,
|
||||
self.remote.database_dir,
|
||||
self.remote.pdf_dir,
|
||||
)
|
||||
if any(not value.strip() for value in values):
|
||||
raise ValueError("公開サーバ設定が不足しています")
|
||||
@@ -0,0 +1,47 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
ENV_NAME = "PAPER_DB_CONFIG"
|
||||
|
||||
COMMON_ROOT = Path(__file__).resolve().parent
|
||||
if str(COMMON_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(COMMON_ROOT))
|
||||
|
||||
|
||||
def resolve_config_path(value: str | Path | None = None) -> Path:
|
||||
raw = value or os.environ.get(ENV_NAME)
|
||||
path = Path(raw).expanduser() if raw else Path.cwd() / "config.py"
|
||||
path = path.resolve()
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"config.pyがありません: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def load_module(path: Path) -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"paper_db_project_config", path
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(f"config.pyを読み込めません: {path}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def load_config(value: str | Path | None = None) -> tuple[Path, Any]:
|
||||
path = resolve_config_path(value)
|
||||
module = load_module(path)
|
||||
if not hasattr(module, "CONFIG"):
|
||||
raise AttributeError(f"CONFIGがありません: {path}")
|
||||
config = module.CONFIG
|
||||
if Path(config.path.project_root).resolve() != path.parent:
|
||||
raise ValueError(
|
||||
"project_rootはconfig.pyのディレクトリにしてください"
|
||||
)
|
||||
return path, config
|
||||
Executable
+323
@@ -0,0 +1,323 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
from project_loader import load_config
|
||||
|
||||
COMMON_ROOT = Path(__file__).resolve().parent
|
||||
SCRIPTS_DIR = COMMON_ROOT / "scripts"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeOptions:
|
||||
mode: str
|
||||
backend: str
|
||||
max_papers: int
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
number = int(value)
|
||||
if number < 1:
|
||||
raise argparse.ArgumentTypeError("1以上を指定してください")
|
||||
return number
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="論文解析、DB整理、公開処理を実行する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"mode",
|
||||
choices=("process", "sync", "publish", "all"),
|
||||
help="実行モード",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
help="省略時はカレントディレクトリのconfig.py",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
choices=("claude", "codex"),
|
||||
help=(
|
||||
"使用するAIバックエンド。省略時は"
|
||||
"config.pyの設定を使用"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-papers",
|
||||
type=positive_int,
|
||||
metavar="N",
|
||||
help=(
|
||||
"1回に処理する最大論文数。省略時は"
|
||||
"config.pyの設定を使用"
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def run_command(
|
||||
command: Sequence[object],
|
||||
*,
|
||||
config_path: Path,
|
||||
config: object,
|
||||
check: bool = True,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
args = [str(item) for item in command]
|
||||
print("+", " ".join(args))
|
||||
env = os.environ.copy()
|
||||
env["PAPER_DB_CONFIG"] = str(config_path)
|
||||
return subprocess.run(
|
||||
args,
|
||||
cwd=config.path.project_root,
|
||||
env=env,
|
||||
text=True,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def require_commands(names: Sequence[str]) -> None:
|
||||
for name in names:
|
||||
if shutil.which(name) is None:
|
||||
raise RuntimeError(
|
||||
f"必要なコマンドが見つかりません: {name}"
|
||||
)
|
||||
|
||||
|
||||
def script(name: str) -> Path:
|
||||
path = SCRIPTS_DIR / name
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
return path
|
||||
|
||||
|
||||
def ensure_database(config_path: Path, config: object) -> None:
|
||||
if config.path.database.is_file():
|
||||
return
|
||||
print("=== データベース初期化 ===")
|
||||
run_command(
|
||||
[sys.executable, script("init_db.py"), config.path.database],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def prepare_work_directories(config: object) -> None:
|
||||
shutil.rmtree(config.path.tmp_dir, ignore_errors=True)
|
||||
config.path.text_dir.mkdir(parents=True, exist_ok=True)
|
||||
config.path.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def extract_text(
|
||||
config_path: Path, config: object, max_papers: int
|
||||
) -> bool:
|
||||
p = config.processing
|
||||
command: list[object] = [
|
||||
sys.executable, script("extract_text.py"),
|
||||
"--database", config.path.database,
|
||||
"--pdf-root", config.path.pdf_root,
|
||||
"--output-dir", config.path.text_dir,
|
||||
"--limit", max_papers,
|
||||
"--pages", p.initial_pages,
|
||||
"--fallback-pages", p.fallback_pages,
|
||||
"--min-text-length", p.min_text_length,
|
||||
]
|
||||
for directory in p.scan_directories:
|
||||
command.extend(["--scan-dir", directory])
|
||||
result = run_command(
|
||||
command,
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return True
|
||||
if result.returncode == 10:
|
||||
print("=== 未登録PDFはありません ===")
|
||||
return False
|
||||
raise RuntimeError(
|
||||
f"extract_text.pyが失敗しました: {result.returncode}"
|
||||
)
|
||||
|
||||
|
||||
def process_new_papers(
|
||||
config_path: Path, config: object, options: RuntimeOptions
|
||||
) -> None:
|
||||
config.validate()
|
||||
require_commands(("pdftotext", options.backend))
|
||||
ensure_database(config_path, config)
|
||||
prepare_work_directories(config)
|
||||
if not extract_text(config_path, config, options.max_papers):
|
||||
return
|
||||
run_command(
|
||||
[sys.executable, script("run_ai.py"),
|
||||
"--backend", options.backend,
|
||||
"--max-papers", options.max_papers],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
if config.processing.update_database:
|
||||
run_command(
|
||||
[sys.executable, script("update_all.py"),
|
||||
"--input-dir", config.path.tmp_dir,
|
||||
"--database", config.path.database,
|
||||
"--pdf-root", config.path.pdf_root,
|
||||
"--continue-on-error"],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def sync_local_database(config_path: Path, config: object) -> None:
|
||||
"""PDFパス更新、DB検証、Org出力を実行する。"""
|
||||
config.validate()
|
||||
ensure_database(config_path, config)
|
||||
|
||||
run_command(
|
||||
[
|
||||
sys.executable,
|
||||
script("update_pdf_paths.py"),
|
||||
"--database",
|
||||
config.path.database,
|
||||
"--pdf-root",
|
||||
config.path.pdf_root,
|
||||
"--apply",
|
||||
],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
if config.processing.validate_database:
|
||||
run_command(
|
||||
[
|
||||
sys.executable,
|
||||
script("validate_db.py"),
|
||||
config.path.database,
|
||||
"--pdf-root",
|
||||
config.path.pdf_root,
|
||||
],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
if config.processing.export_org:
|
||||
run_command(
|
||||
[
|
||||
sys.executable,
|
||||
script("export_org.py"),
|
||||
config.path.database,
|
||||
"-o",
|
||||
config.path.papers_org,
|
||||
],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def publish_remote(config_path: Path, config: object) -> None:
|
||||
"""PDFと公開用DBをリモートへ反映する。"""
|
||||
config.validate(require_remote=True)
|
||||
require_commands(("rsync", "scp", "ssh"))
|
||||
if not config.path.database.is_file():
|
||||
raise FileNotFoundError(config.path.database)
|
||||
sync_pdfs(config_path, config)
|
||||
publish_database(config_path, config)
|
||||
|
||||
|
||||
def create_publish_database(config: object) -> None:
|
||||
config.path.publish_database.unlink(missing_ok=True)
|
||||
with sqlite3.connect(config.path.database) as source:
|
||||
with sqlite3.connect(config.path.publish_database) as target:
|
||||
source.backup(target)
|
||||
with sqlite3.connect(config.path.publish_database) as con:
|
||||
con.execute("PRAGMA journal_mode=DELETE")
|
||||
con.commit()
|
||||
|
||||
|
||||
def shell_quote(value: str) -> str:
|
||||
return shlex.quote(value)
|
||||
|
||||
|
||||
def publish_database(config_path: Path, config: object) -> None:
|
||||
config.validate(require_remote=True)
|
||||
require_commands(("scp", "ssh"))
|
||||
if not config.path.database.is_file():
|
||||
raise FileNotFoundError(config.path.database)
|
||||
create_publish_database(config)
|
||||
destination = (
|
||||
f"{config.remote.host}:"
|
||||
f"{config.remote.database_dir}/papers.db.new"
|
||||
)
|
||||
run_command(
|
||||
["scp", config.path.publish_database, destination],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
remote_dir = shell_quote(config.remote.database_dir)
|
||||
remote_script = f"""set -eu
|
||||
cd {remote_dir}
|
||||
sqlite3 papers.db.new 'PRAGMA integrity_check;'
|
||||
mv -f papers.db.new papers.db
|
||||
sqlite3 papers.db 'PRAGMA integrity_check;'
|
||||
"""
|
||||
run_command(
|
||||
["ssh", config.remote.host, remote_script],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def sync_pdfs(config_path: Path, config: object) -> None:
|
||||
config.validate(require_remote=True)
|
||||
require_commands(("rsync",))
|
||||
run_command(
|
||||
["rsync", "-az", f"{config.path.pdf_root}/",
|
||||
f"{config.remote.host}:{config.remote.pdf_dir}/"],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
config_path, config = load_config(args.config)
|
||||
options = RuntimeOptions(
|
||||
mode=args.mode,
|
||||
backend=args.backend or config.ai.backend,
|
||||
max_papers=(
|
||||
args.max_papers
|
||||
or config.processing.max_papers
|
||||
),
|
||||
)
|
||||
|
||||
if options.mode in {"process", "all"}:
|
||||
process_new_papers(config_path, config, options)
|
||||
|
||||
if options.mode in {"sync", "all"}:
|
||||
sync_local_database(config_path, config)
|
||||
|
||||
if options.mode in {"publish", "all"}:
|
||||
publish_remote(config_path, config)
|
||||
|
||||
print("=== 完了 ===")
|
||||
return 0
|
||||
except KeyboardInterrupt:
|
||||
print("中断しました。", file=sys.stderr)
|
||||
return 130
|
||||
except Exception as exc:
|
||||
print(f"publish.py: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import html
|
||||
import re
|
||||
import unicodedata
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import sys
|
||||
|
||||
COMMON_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(COMMON_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(COMMON_ROOT))
|
||||
|
||||
from project_loader import load_config # noqa: E402
|
||||
|
||||
_, CONFIG = load_config()
|
||||
ALLOWED_PRIORITIES = set(CONFIG.domain.priorities)
|
||||
ALLOWED_CATEGORIES = set(CONFIG.domain.categories)
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
def normalize_doi(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
doi = str(value).strip().lower()
|
||||
if not doi or doi == "unknown":
|
||||
return None
|
||||
prefixes = (
|
||||
"https://doi.org/",
|
||||
"http://doi.org/",
|
||||
"http://dx.doi.org/",
|
||||
"doi:",
|
||||
)
|
||||
for prefix in prefixes:
|
||||
if doi.startswith(prefix):
|
||||
doi = doi[len(prefix):]
|
||||
break
|
||||
doi = doi.strip().rstrip(".,;:)]}")
|
||||
return doi or None
|
||||
|
||||
def normalize_text(value: str) -> str:
|
||||
value = html.unescape(value)
|
||||
value = unicodedata.normalize("NFKC", value)
|
||||
value = value.lower()
|
||||
value = re.sub(r"[\r\n\t]+", " ", value)
|
||||
value = re.sub(r"[^\w\s]", " ", value, flags=re.UNICODE)
|
||||
value = re.sub(r"\s+", " ", value).strip()
|
||||
return value
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for block in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
def resolve_pdf_path(pdf_root: Path, stored_path: str) -> Path:
|
||||
path = Path(stored_path)
|
||||
if path.is_absolute():
|
||||
raise ValueError("pdf_path must be relative to PDF_ROOT")
|
||||
resolved = (pdf_root / path).resolve()
|
||||
root = pdf_root.resolve()
|
||||
try:
|
||||
resolved.relative_to(root)
|
||||
except ValueError as exc:
|
||||
raise ValueError("pdf_path escapes PDF_ROOT") from exc
|
||||
return resolved
|
||||
Executable
+269
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
"""paper_idを指定してpapers.dbから論文を削除する。
|
||||
|
||||
PDFファイルそのものは削除しない。
|
||||
既定はdry-run。実際に削除する場合は--applyを指定する。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Paper:
|
||||
paper_id: int
|
||||
title: str
|
||||
first_author: str | None
|
||||
year: int | None
|
||||
doi: str | None
|
||||
pdf_path: str | None
|
||||
file_sha256: str | None
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="paper_idを指定してDBレコードを削除する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"paper_id",
|
||||
type=int,
|
||||
help="削除するpapers.id",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database",
|
||||
type=Path,
|
||||
default=CONFIG.path.database,
|
||||
help="SQLiteデータベース",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="削除を実際にDBへ反映する",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-backup",
|
||||
action="store_true",
|
||||
help="削除前バックアップを作成しない",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def open_database(
|
||||
database: Path,
|
||||
) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
|
||||
def load_paper(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
) -> Paper | None:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT
|
||||
id,
|
||||
title,
|
||||
first_author,
|
||||
year,
|
||||
doi,
|
||||
pdf_path,
|
||||
file_sha256
|
||||
FROM papers
|
||||
WHERE id = ?
|
||||
""",
|
||||
(paper_id,),
|
||||
).fetchone()
|
||||
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
return Paper(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
first_author=row["first_author"],
|
||||
year=row["year"],
|
||||
doi=row["doi"],
|
||||
pdf_path=row["pdf_path"],
|
||||
file_sha256=row["file_sha256"],
|
||||
)
|
||||
|
||||
|
||||
def print_paper(paper: Paper) -> None:
|
||||
print("=== 削除対象 ===")
|
||||
print(f"id: {paper.paper_id}")
|
||||
print(f"title: {paper.title}")
|
||||
print(
|
||||
"first_author: "
|
||||
f"{paper.first_author or '(none)'}"
|
||||
)
|
||||
print(f"year: {paper.year or '(none)'}")
|
||||
print(f"doi: {paper.doi or '(none)'}")
|
||||
print(
|
||||
f"pdf_path: {paper.pdf_path or '(none)'}"
|
||||
)
|
||||
print(
|
||||
"file_sha256: "
|
||||
f"{paper.file_sha256 or '(none)'}"
|
||||
)
|
||||
|
||||
|
||||
def create_backup(database: Path) -> Path:
|
||||
backup = database.with_name(
|
||||
database.name + ".before-delete"
|
||||
)
|
||||
|
||||
counter = 1
|
||||
|
||||
while backup.exists():
|
||||
backup = database.with_name(
|
||||
database.name + f".before-delete.{counter}"
|
||||
)
|
||||
counter += 1
|
||||
|
||||
shutil.copy2(database, backup)
|
||||
return backup
|
||||
|
||||
|
||||
def delete_paper(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
) -> None:
|
||||
"""関連レコードを含めて1トランザクションで削除する。"""
|
||||
with connection:
|
||||
tables = (
|
||||
"paper_authors",
|
||||
"paper_keywords",
|
||||
"processing_log",
|
||||
)
|
||||
|
||||
for table in tables:
|
||||
if table_exists(connection, table):
|
||||
connection.execute(
|
||||
f"DELETE FROM {table} WHERE paper_id = ?",
|
||||
(paper_id,),
|
||||
)
|
||||
|
||||
cursor = connection.execute(
|
||||
"DELETE FROM papers WHERE id = ?",
|
||||
(paper_id,),
|
||||
)
|
||||
|
||||
if cursor.rowcount != 1:
|
||||
raise RuntimeError(
|
||||
f"papers.id={paper_id}を削除できませんでした"
|
||||
)
|
||||
|
||||
|
||||
def table_exists(
|
||||
connection: sqlite3.Connection,
|
||||
table: str,
|
||||
) -> bool:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT 1
|
||||
FROM sqlite_master
|
||||
WHERE type = 'table'
|
||||
AND name = ?
|
||||
""",
|
||||
(table,),
|
||||
).fetchone()
|
||||
|
||||
return row is not None
|
||||
|
||||
|
||||
def validate_database(
|
||||
connection: sqlite3.Connection,
|
||||
) -> None:
|
||||
integrity = connection.execute(
|
||||
"PRAGMA integrity_check"
|
||||
).fetchone()[0]
|
||||
|
||||
if integrity != "ok":
|
||||
raise RuntimeError(
|
||||
f"integrity_check: {integrity}"
|
||||
)
|
||||
|
||||
foreign_keys = connection.execute(
|
||||
"PRAGMA foreign_key_check"
|
||||
).fetchall()
|
||||
|
||||
if foreign_keys:
|
||||
raise RuntimeError(
|
||||
"foreign_key_check: "
|
||||
f"{len(foreign_keys)}件"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
try:
|
||||
database = args.database.expanduser().resolve()
|
||||
|
||||
if not database.is_file():
|
||||
raise FileNotFoundError(database)
|
||||
|
||||
with open_database(database) as connection:
|
||||
paper = load_paper(
|
||||
connection,
|
||||
args.paper_id,
|
||||
)
|
||||
|
||||
if paper is None:
|
||||
print(
|
||||
f"paper_id={args.paper_id}は存在しません。",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
print_paper(paper)
|
||||
|
||||
if not args.apply:
|
||||
print()
|
||||
print(
|
||||
"削除するには --apply を指定してください。"
|
||||
)
|
||||
return 0
|
||||
|
||||
if not args.no_backup:
|
||||
backup = create_backup(database)
|
||||
print()
|
||||
print(f"DBバックアップ: {backup}")
|
||||
|
||||
delete_paper(
|
||||
connection,
|
||||
args.paper_id,
|
||||
)
|
||||
validate_database(connection)
|
||||
|
||||
print()
|
||||
print(
|
||||
f"Deleted paper_id={args.paper_id}"
|
||||
)
|
||||
print("PDFファイルは削除していません。")
|
||||
return 0
|
||||
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"delete_paper.py: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+205
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Sequence, TextIO
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
|
||||
def prop(value: object) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
return str(value).replace("\n", " ").strip()
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"database",
|
||||
nargs="?",
|
||||
default=str(CONFIG.path.database),
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
help="出力先。省略時は標準出力",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def open_database(database: str) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
return connection
|
||||
|
||||
|
||||
def load_papers(
|
||||
connection: sqlite3.Connection,
|
||||
) -> list[sqlite3.Row]:
|
||||
query = """
|
||||
SELECT p.*, c.name AS category
|
||||
FROM papers AS p
|
||||
LEFT JOIN categories AS c
|
||||
ON c.id = p.category_id
|
||||
ORDER BY
|
||||
COALESCE(p.year, 0) DESC,
|
||||
p.first_author,
|
||||
p.title
|
||||
"""
|
||||
return connection.execute(query).fetchall()
|
||||
|
||||
|
||||
def load_authors(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
) -> list[str]:
|
||||
query = """
|
||||
SELECT a.name
|
||||
FROM paper_authors AS pa
|
||||
JOIN authors AS a
|
||||
ON a.id = pa.author_id
|
||||
WHERE pa.paper_id = ?
|
||||
ORDER BY pa.author_order
|
||||
"""
|
||||
rows = connection.execute(query, (paper_id,)).fetchall()
|
||||
return [row["name"] for row in rows]
|
||||
|
||||
|
||||
def load_keywords(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
) -> list[str]:
|
||||
query = """
|
||||
SELECT k.name
|
||||
FROM paper_keywords AS pk
|
||||
JOIN keywords AS k
|
||||
ON k.id = pk.keyword_id
|
||||
WHERE pk.paper_id = ?
|
||||
ORDER BY k.name_normalized
|
||||
"""
|
||||
rows = connection.execute(query, (paper_id,)).fetchall()
|
||||
return [row["name"] for row in rows]
|
||||
|
||||
|
||||
def make_header() -> list[str]:
|
||||
exported_at = (
|
||||
datetime.now(timezone.utc)
|
||||
.astimezone()
|
||||
.isoformat(timespec="seconds")
|
||||
)
|
||||
return [
|
||||
"#+TITLE: ガラス固化体論文データベース",
|
||||
f"#+DB_EXPORTED_AT: {exported_at}",
|
||||
"",
|
||||
(
|
||||
"# 編集可能: CATEGORY, RELEVANCE, PRIORITY, STATUS, "
|
||||
"日本語要約, キーワード, メモ"
|
||||
),
|
||||
(
|
||||
"# 保護対象: DB_ID, DOI, TITLE, AUTHORS, YEAR, "
|
||||
"JOURNAL, PDF_PATH, FILE_SHA256"
|
||||
),
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
def make_properties(
|
||||
paper: sqlite3.Row,
|
||||
authors: Sequence[str],
|
||||
) -> list[str]:
|
||||
category = paper["category"] or "Other"
|
||||
return [
|
||||
":PROPERTIES:",
|
||||
f":DB_ID: {paper['id']}",
|
||||
f":DB_UPDATED_AT: {prop(paper['updated_at'])}",
|
||||
f":DOI: {prop(paper['doi'])}",
|
||||
f":TITLE: {prop(paper['title'])}",
|
||||
f":AUTHORS: {prop('; '.join(authors))}",
|
||||
f":FIRST_AUTHOR: {prop(paper['first_author'])}",
|
||||
f":YEAR: {prop(paper['year'])}",
|
||||
f":JOURNAL: {prop(paper['journal'])}",
|
||||
f":PDF_PATH: {prop(paper['pdf_path'])}",
|
||||
f":FILE_SHA256: {prop(paper['file_sha256'])}",
|
||||
f":CATEGORY: {prop(category)}",
|
||||
f":RELEVANCE: {prop(paper['relevance'])}",
|
||||
f":PRIORITY: {prop(paper['priority'])}",
|
||||
f":STATUS: {prop(paper['review_status'])}",
|
||||
":END:",
|
||||
]
|
||||
|
||||
|
||||
def make_paper_entry(
|
||||
paper: sqlite3.Row,
|
||||
authors: Sequence[str],
|
||||
keywords: Sequence[str],
|
||||
) -> list[str]:
|
||||
paper_id = paper["id"]
|
||||
title = prop(paper["title"])
|
||||
lines = [
|
||||
f"* [ID:{paper_id}] {title}",
|
||||
*make_properties(paper, authors),
|
||||
"",
|
||||
"** 日本語要約",
|
||||
prop(paper["summary_ja"]),
|
||||
"",
|
||||
"** キーワード",
|
||||
]
|
||||
lines.extend(f"- {keyword}" for keyword in keywords)
|
||||
lines.extend(["", "** メモ", prop(paper["notes"]), ""])
|
||||
return lines
|
||||
|
||||
|
||||
def build_org(
|
||||
connection: sqlite3.Connection,
|
||||
papers: Sequence[sqlite3.Row],
|
||||
) -> str:
|
||||
lines = make_header()
|
||||
for paper in papers:
|
||||
paper_id = paper["id"]
|
||||
authors = load_authors(connection, paper_id)
|
||||
keywords = load_keywords(connection, paper_id)
|
||||
lines.extend(make_paper_entry(paper, authors, keywords))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_output(content: str, output: str | None) -> None:
|
||||
if output is None:
|
||||
sys.stdout.write(content)
|
||||
if not content.endswith("\n"):
|
||||
sys.stdout.write("\n")
|
||||
return
|
||||
|
||||
Path(output).write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def print_summary(
|
||||
paper_count: int,
|
||||
output: str | None,
|
||||
stream: TextIO = sys.stderr,
|
||||
) -> None:
|
||||
destination = output if output is not None else "stdout"
|
||||
print(
|
||||
f"Exported {paper_count} papers to {destination}",
|
||||
file=stream,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
with open_database(args.database) as connection:
|
||||
papers = load_papers(connection)
|
||||
content = build_org(connection, papers)
|
||||
|
||||
write_output(content, args.output)
|
||||
print_summary(len(papers), args.output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+1137
File diff suppressed because it is too large
Load Diff
Executable
+161
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
CATEGORIES = CONFIG.domain.categories
|
||||
|
||||
SCHEMA = """
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS categories (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS papers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
doi TEXT,
|
||||
title TEXT NOT NULL,
|
||||
title_normalized TEXT NOT NULL,
|
||||
first_author TEXT,
|
||||
year INTEGER,
|
||||
journal TEXT,
|
||||
volume TEXT,
|
||||
issue TEXT,
|
||||
pages TEXT,
|
||||
publisher TEXT,
|
||||
article_type TEXT,
|
||||
language TEXT,
|
||||
abstract TEXT,
|
||||
summary_ja TEXT,
|
||||
relevance INTEGER NOT NULL DEFAULT 0 CHECK (relevance BETWEEN 0 AND 5),
|
||||
priority TEXT NOT NULL DEFAULT 'D' CHECK (priority IN ('A','B','C','D')),
|
||||
review_status TEXT NOT NULL DEFAULT 'unreviewed',
|
||||
category_id INTEGER,
|
||||
pdf_path TEXT,
|
||||
file_size INTEGER,
|
||||
file_mtime TEXT,
|
||||
file_sha256 TEXT,
|
||||
source TEXT,
|
||||
landing_url TEXT,
|
||||
pdf_url TEXT,
|
||||
url_source TEXT,
|
||||
access_status TEXT,
|
||||
url_checked_at TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
checked_at TEXT,
|
||||
notes TEXT,
|
||||
FOREIGN KEY (category_id) REFERENCES categories(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS authors (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
name_normalized TEXT NOT NULL,
|
||||
orcid TEXT,
|
||||
affiliation TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS paper_authors (
|
||||
paper_id INTEGER NOT NULL,
|
||||
author_id INTEGER NOT NULL,
|
||||
author_order INTEGER NOT NULL,
|
||||
is_corresponding INTEGER NOT NULL DEFAULT 0 CHECK (is_corresponding IN (0,1)),
|
||||
PRIMARY KEY (paper_id, author_order),
|
||||
UNIQUE (paper_id, author_id),
|
||||
FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (author_id) REFERENCES authors(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS keywords (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
name_normalized TEXT NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS paper_keywords (
|
||||
paper_id INTEGER NOT NULL,
|
||||
keyword_id INTEGER NOT NULL,
|
||||
PRIMARY KEY (paper_id, keyword_id),
|
||||
FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (keyword_id) REFERENCES keywords(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS processing_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
pdf_path TEXT,
|
||||
paper_id INTEGER,
|
||||
status TEXT NOT NULL CHECK (status IN ('registered','updated','duplicate','skipped','error')),
|
||||
message TEXT,
|
||||
processed_at TEXT NOT NULL,
|
||||
FOREIGN KEY (paper_id) REFERENCES papers(id) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_papers_doi
|
||||
ON papers(doi) WHERE doi IS NOT NULL AND doi <> '';
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_papers_sha256
|
||||
ON papers(file_sha256) WHERE file_sha256 IS NOT NULL AND file_sha256 <> '';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_title_normalized ON papers(title_normalized);
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_year ON papers(year);
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_relevance ON papers(relevance);
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_priority ON papers(priority);
|
||||
CREATE INDEX IF NOT EXISTS idx_papers_category_id ON papers(category_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_authors_name_normalized ON authors(name_normalized);
|
||||
CREATE INDEX IF NOT EXISTS idx_keywords_name_normalized ON keywords(name_normalized);
|
||||
"""
|
||||
|
||||
URL_COLUMNS = {
|
||||
"landing_url": "TEXT",
|
||||
"pdf_url": "TEXT",
|
||||
"url_source": "TEXT",
|
||||
"access_status": "TEXT",
|
||||
"url_checked_at": "TEXT",
|
||||
}
|
||||
|
||||
|
||||
def ensure_url_columns(con: sqlite3.Connection) -> None:
|
||||
columns = {
|
||||
row[1] for row in con.execute("PRAGMA table_info(papers)")
|
||||
}
|
||||
for name, column_type in URL_COLUMNS.items():
|
||||
if name not in columns:
|
||||
con.execute(
|
||||
f'ALTER TABLE papers ADD COLUMN "{name}" {column_type}'
|
||||
)
|
||||
|
||||
|
||||
def initialize(path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with sqlite3.connect(path) as con:
|
||||
con.executescript(SCHEMA)
|
||||
ensure_url_columns(con)
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO categories(name, description) VALUES (?, NULL)",
|
||||
[(name,) for name in CATEGORIES],
|
||||
)
|
||||
result = con.execute("PRAGMA integrity_check").fetchone()[0]
|
||||
if result != "ok":
|
||||
raise RuntimeError(result)
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument(
|
||||
"database", nargs="?", default=str(CONFIG.path.database)
|
||||
)
|
||||
args = p.parse_args()
|
||||
initialize(Path(args.database))
|
||||
print(f"Initialized: {args.database}")
|
||||
print("integrity_check: ok")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""config.py の設定値を表示する互換ユーティリティ。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
data = {
|
||||
"PDF_ROOT": str(CONFIG.path.pdf_root),
|
||||
"SCAN_DIRECTORIES": list(
|
||||
CONFIG.processing.scan_directories
|
||||
),
|
||||
"REMOTE": CONFIG.remote.host,
|
||||
"REMOTE_DB": CONFIG.remote.database_dir,
|
||||
"REMOTE_PDF": CONFIG.remote.pdf_dir,
|
||||
"MAX_PAPERS": CONFIG.processing.max_papers,
|
||||
"AI_BACKEND": CONFIG.ai.backend,
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
for key, value in data.items():
|
||||
print(f"{key}={value}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+220
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""設定に応じてClaude CodeまたはCodex CLIを実行する。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from common import CONFIG
|
||||
|
||||
|
||||
COMMON_ROOT = Path(__file__).resolve().parents[1]
|
||||
INSTRUCTIONS_FILE = COMMON_ROOT / "CLAUDE.md"
|
||||
|
||||
AI_PROMPT = """
|
||||
以下の共通指示に従い、TASK=process_new_papersを
|
||||
追加確認なしで最後まで実行してください。
|
||||
|
||||
--- 共通指示開始 ---
|
||||
{common_instructions}
|
||||
--- 共通指示終了 ---
|
||||
|
||||
tmp/text/には、未登録PDFから抽出したテキストがあります。
|
||||
|
||||
対象分野:
|
||||
{domain_name}
|
||||
|
||||
対象分野の説明:
|
||||
{domain_description}
|
||||
|
||||
categoryは次のいずれかを使用してください。
|
||||
{categories}
|
||||
|
||||
日本語要約は{summary_length_ja}を目安とし、
|
||||
次の観点を優先してください。
|
||||
{summary_focus}
|
||||
|
||||
relevanceは次の基準で判定してください。
|
||||
{relevance_criteria}
|
||||
|
||||
Priorityは次の対応にしてください。
|
||||
- A: relevance=5
|
||||
- B: relevance=4
|
||||
- C: relevance=3
|
||||
- D: relevance=0〜2
|
||||
|
||||
行う処理は次だけです。
|
||||
|
||||
- tmp/text/にあるテキストファイルを解析する
|
||||
- 各テキスト先頭のPDF_PATHをJSONのpdf_pathに使う
|
||||
- DOI、書誌情報、要約、キーワード、分類を抽出する
|
||||
- DOIがある場合はlanding_urlを生成する
|
||||
- 必要最小限のWeb検索で公開PDFを確認する
|
||||
- 各論文についてtmp/paper-*.jsonを作成する
|
||||
- 作成件数と結果を日本語で報告する
|
||||
|
||||
原則としてPDFは直接読まないでください。
|
||||
|
||||
次の場合のみPDF_PATHに記載された元PDFを参照できます。
|
||||
|
||||
- PDF_TEXT_EMPTYがyes
|
||||
- TEXT_INSUFFICIENTがyes
|
||||
- TEXT_LENGTHが{min_text_length}未満
|
||||
- テキストが文字化けしている
|
||||
- タイトル、著者、発行年、要約を確認できない
|
||||
|
||||
元PDFを参照する場合も、PDF_PATHに記載された1ファイルだけを読み、
|
||||
他のPDFやディレクトリを探索しないでください。
|
||||
|
||||
確認できない内容、DOI、pdf_urlは推測しないでください。
|
||||
papers.dbの直接更新、DB検証、Org出力は実行しないでください。
|
||||
最大{max_papers}件だけ処理してください。
|
||||
""".strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeOptions:
|
||||
backend: str
|
||||
max_papers: int
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
number = int(value)
|
||||
if number < 1:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"1以上の整数を指定してください"
|
||||
)
|
||||
return number
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Claude CodeまたはCodex CLIを実行する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
choices=("claude", "codex"),
|
||||
default=None,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-papers",
|
||||
type=positive_int,
|
||||
default=None,
|
||||
metavar="N",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def resolve_options(args: argparse.Namespace) -> RuntimeOptions:
|
||||
return RuntimeOptions(
|
||||
backend=args.backend or CONFIG.ai.backend,
|
||||
max_papers=(
|
||||
args.max_papers
|
||||
if args.max_papers is not None
|
||||
else CONFIG.processing.max_papers
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def load_common_instructions() -> str:
|
||||
if not INSTRUCTIONS_FILE.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"共通CLAUDE.mdがありません: {INSTRUCTIONS_FILE}"
|
||||
)
|
||||
return INSTRUCTIONS_FILE.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def build_prompt(max_papers: int) -> str:
|
||||
domain = CONFIG.domain
|
||||
return AI_PROMPT.format(
|
||||
common_instructions=load_common_instructions(),
|
||||
domain_name=domain.name,
|
||||
domain_description=domain.description,
|
||||
categories=domain.categories_prompt(),
|
||||
summary_length_ja=domain.summary_length_ja,
|
||||
summary_focus=domain.summary_focus_prompt(),
|
||||
relevance_criteria=domain.relevance_prompt(),
|
||||
min_text_length=CONFIG.processing.min_text_length,
|
||||
max_papers=max_papers,
|
||||
)
|
||||
|
||||
|
||||
def build_claude_command() -> list[str]:
|
||||
command = [
|
||||
"claude",
|
||||
"--print",
|
||||
"--add-dir",
|
||||
str(CONFIG.path.text_dir),
|
||||
"--permission-mode",
|
||||
CONFIG.ai.claude_permission_mode,
|
||||
"--allowedTools",
|
||||
CONFIG.ai.claude_allowed_tools_argument,
|
||||
]
|
||||
for path in CONFIG.scan_paths:
|
||||
command.extend(["--add-dir", str(path)])
|
||||
return command
|
||||
|
||||
|
||||
def build_codex_command(max_papers: int) -> list[str]:
|
||||
command = [
|
||||
"codex",
|
||||
"exec",
|
||||
"--cd",
|
||||
str(CONFIG.path.project_root),
|
||||
"--sandbox",
|
||||
CONFIG.ai.codex_sandbox,
|
||||
]
|
||||
if CONFIG.ai.codex_model:
|
||||
command.extend(["--model", CONFIG.ai.codex_model])
|
||||
if CONFIG.ai.codex_full_auto:
|
||||
command.append("--full-auto")
|
||||
command.append(build_prompt(max_papers))
|
||||
return command
|
||||
|
||||
|
||||
def build_command(options: RuntimeOptions) -> list[str]:
|
||||
if options.backend == "claude":
|
||||
return build_claude_command()
|
||||
if options.backend == "codex":
|
||||
return build_codex_command(options.max_papers)
|
||||
raise ValueError(f"未対応のAIバックエンドです: {options.backend}")
|
||||
|
||||
|
||||
def require_backend_command(backend: str) -> None:
|
||||
executable = {"claude": "claude", "codex": "codex"}.get(backend)
|
||||
if executable is None:
|
||||
raise ValueError(f"未対応のAIバックエンドです: {backend}")
|
||||
if shutil.which(executable) is None:
|
||||
raise RuntimeError(
|
||||
f"必要なコマンドが見つかりません: {executable}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
options = resolve_options(args)
|
||||
try:
|
||||
CONFIG.validate()
|
||||
require_backend_command(options.backend)
|
||||
command = build_command(options)
|
||||
prompt = build_prompt(options.max_papers)
|
||||
completed = subprocess.run(
|
||||
command,
|
||||
cwd=CONFIG.path.project_root,
|
||||
input=(prompt if options.backend == "claude" else None),
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return completed.returncode
|
||||
except Exception as exc:
|
||||
print(f"run_ai.py: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+542
@@ -0,0 +1,542 @@
|
||||
#!/usr/bin/env python3
|
||||
"""tmp/paper-*.jsonをpapers.dbへ一括登録する。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from common import (
|
||||
ALLOWED_CATEGORIES,
|
||||
ALLOWED_PRIORITIES,
|
||||
CONFIG,
|
||||
normalize_doi,
|
||||
normalize_text,
|
||||
now_iso,
|
||||
resolve_pdf_path,
|
||||
sha256_file,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""コマンドライン引数を解析する。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="tmp/paper-*.jsonをpapers.dbへ一括登録する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-dir",
|
||||
default=str(CONFIG.path.tmp_dir),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database",
|
||||
default=str(CONFIG.path.database),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pdf-root",
|
||||
default=str(CONFIG.path.pdf_root),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pattern",
|
||||
default="paper-*.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--continue-on-error",
|
||||
action="store_true",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def backup_database(database: Path) -> Path:
|
||||
"""DBを1回だけバックアップする。"""
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
backup = database.with_name(
|
||||
f"{database.name}.bak-{stamp}"
|
||||
)
|
||||
shutil.copy2(database, backup)
|
||||
return backup
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
"""論文JSONを読み込む。"""
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("JSON root must be an object")
|
||||
return data
|
||||
|
||||
|
||||
def validate_json(data: dict[str, Any]) -> None:
|
||||
"""JSONの必須項目と型を検証する。"""
|
||||
required = ("title", "authors", "year", "journal", "pdf_path")
|
||||
for key in required:
|
||||
if key not in data:
|
||||
raise ValueError(f"missing required field: {key}")
|
||||
|
||||
authors = data["authors"]
|
||||
if not isinstance(authors, list) or not authors:
|
||||
raise ValueError("authors must be a non-empty list")
|
||||
|
||||
relevance = int(data.get("relevance", 0))
|
||||
if relevance not in range(6):
|
||||
raise ValueError("relevance must be 0..5")
|
||||
|
||||
priority = data.get("priority", "D")
|
||||
if priority not in ALLOWED_PRIORITIES:
|
||||
raise ValueError("priority must be A..D")
|
||||
|
||||
category = data.get("category", "Other")
|
||||
if category not in ALLOWED_CATEGORIES:
|
||||
raise ValueError("unknown category")
|
||||
|
||||
|
||||
def add_file_metadata(
|
||||
data: dict[str, Any],
|
||||
pdf_root: Path,
|
||||
) -> None:
|
||||
"""ローカルPDFの情報をJSONへ追加する。"""
|
||||
actual_pdf = resolve_pdf_path(
|
||||
pdf_root,
|
||||
str(data["pdf_path"]),
|
||||
)
|
||||
if not actual_pdf.is_file():
|
||||
raise FileNotFoundError(actual_pdf)
|
||||
|
||||
stat = actual_pdf.stat()
|
||||
data["file_size"] = stat.st_size
|
||||
data["file_mtime"] = datetime.fromtimestamp(
|
||||
stat.st_mtime
|
||||
).astimezone().isoformat(timespec="seconds")
|
||||
data["file_sha256"] = sha256_file(actual_pdf)
|
||||
|
||||
|
||||
def get_category_id(
|
||||
connection: sqlite3.Connection,
|
||||
name: str,
|
||||
) -> int:
|
||||
"""カテゴリIDを取得する。"""
|
||||
connection.execute(
|
||||
"INSERT OR IGNORE INTO categories(name) VALUES (?)",
|
||||
(name,),
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT id FROM categories WHERE name = ?",
|
||||
(name,),
|
||||
).fetchone()
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def get_author_id(
|
||||
connection: sqlite3.Connection,
|
||||
author: dict[str, Any],
|
||||
) -> int:
|
||||
"""著者を取得または登録する。"""
|
||||
name = str(author.get("name", "")).strip()
|
||||
normalized = normalize_text(name)
|
||||
orcid = author.get("orcid")
|
||||
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM authors
|
||||
WHERE name_normalized = ?
|
||||
AND ((orcid = ?) OR (orcid IS NULL AND ? IS NULL))
|
||||
ORDER BY id
|
||||
LIMIT 1
|
||||
""",
|
||||
(normalized, orcid, orcid),
|
||||
).fetchone()
|
||||
|
||||
if row:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE authors
|
||||
SET affiliation = COALESCE(affiliation, ?),
|
||||
orcid = COALESCE(orcid, ?)
|
||||
WHERE id = ?
|
||||
""",
|
||||
(author.get("affiliation"), orcid, row[0]),
|
||||
)
|
||||
return int(row[0])
|
||||
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO authors(
|
||||
name, name_normalized, orcid, affiliation
|
||||
) VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(name, normalized, orcid, author.get("affiliation")),
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
|
||||
def get_keyword_id(
|
||||
connection: sqlite3.Connection,
|
||||
keyword: Any,
|
||||
) -> int:
|
||||
"""キーワードIDを取得する。"""
|
||||
name = " ".join(str(keyword).split())
|
||||
normalized = normalize_text(name)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO keywords(name, name_normalized)
|
||||
VALUES (?, ?)
|
||||
""",
|
||||
(name, normalized),
|
||||
)
|
||||
row = connection.execute(
|
||||
"SELECT id FROM keywords WHERE name_normalized = ?",
|
||||
(normalized,),
|
||||
).fetchone()
|
||||
return int(row[0])
|
||||
|
||||
|
||||
def find_existing(
|
||||
connection: sqlite3.Connection,
|
||||
data: dict[str, Any],
|
||||
) -> sqlite3.Row | None:
|
||||
"""既存論文をSHA256、DOI、書誌情報の順で探す。"""
|
||||
sha256 = data.get("file_sha256")
|
||||
if sha256:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM papers WHERE file_sha256 = ?",
|
||||
(sha256,),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row
|
||||
|
||||
doi = normalize_doi(data.get("doi"))
|
||||
if doi:
|
||||
row = connection.execute(
|
||||
"SELECT * FROM papers WHERE doi = ?",
|
||||
(doi,),
|
||||
).fetchone()
|
||||
if row:
|
||||
return row
|
||||
|
||||
first_author = (
|
||||
data.get("first_author")
|
||||
or data["authors"][0].get("name", "")
|
||||
)
|
||||
return connection.execute(
|
||||
"""
|
||||
SELECT *
|
||||
FROM papers
|
||||
WHERE title_normalized = ?
|
||||
AND COALESCE(first_author, '') = ?
|
||||
AND year IS ?
|
||||
""",
|
||||
(
|
||||
normalize_text(str(data["title"])),
|
||||
first_author,
|
||||
data.get("year"),
|
||||
),
|
||||
).fetchone()
|
||||
|
||||
|
||||
def update_existing_paper(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
category_id: int,
|
||||
data: dict[str, Any],
|
||||
now: str,
|
||||
) -> None:
|
||||
"""既存論文の空欄を補完する。"""
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE papers SET
|
||||
doi = COALESCE(doi, ?),
|
||||
journal = COALESCE(journal, ?),
|
||||
volume = COALESCE(volume, ?),
|
||||
issue = COALESCE(issue, ?),
|
||||
pages = COALESCE(pages, ?),
|
||||
publisher = COALESCE(publisher, ?),
|
||||
article_type = COALESCE(article_type, ?),
|
||||
language = COALESCE(language, ?),
|
||||
abstract = COALESCE(abstract, ?),
|
||||
summary_ja = COALESCE(summary_ja, ?),
|
||||
category_id = COALESCE(category_id, ?),
|
||||
pdf_path = COALESCE(pdf_path, ?),
|
||||
file_size = COALESCE(file_size, ?),
|
||||
file_mtime = COALESCE(file_mtime, ?),
|
||||
file_sha256 = COALESCE(file_sha256, ?),
|
||||
source = COALESCE(source, ?),
|
||||
landing_url = COALESCE(landing_url, ?),
|
||||
pdf_url = COALESCE(pdf_url, ?),
|
||||
url_source = COALESCE(url_source, ?),
|
||||
access_status = COALESCE(access_status, ?),
|
||||
url_checked_at = COALESCE(url_checked_at, ?),
|
||||
checked_at = ?,
|
||||
updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
normalize_doi(data.get("doi")),
|
||||
data.get("journal"),
|
||||
data.get("volume"),
|
||||
data.get("issue"),
|
||||
data.get("pages"),
|
||||
data.get("publisher"),
|
||||
data.get("article_type"),
|
||||
data.get("language"),
|
||||
data.get("abstract"),
|
||||
data.get("summary_ja"),
|
||||
category_id,
|
||||
data.get("pdf_path"),
|
||||
data["file_size"],
|
||||
data["file_mtime"],
|
||||
data["file_sha256"],
|
||||
data.get("source"),
|
||||
data.get("landing_url"),
|
||||
data.get("pdf_url"),
|
||||
data.get("url_source"),
|
||||
data.get("access_status"),
|
||||
data.get("url_checked_at"),
|
||||
now,
|
||||
now,
|
||||
paper_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def insert_paper(
|
||||
connection: sqlite3.Connection,
|
||||
category_id: int,
|
||||
data: dict[str, Any],
|
||||
now: str,
|
||||
) -> int:
|
||||
"""新規論文を登録する。"""
|
||||
first_author = (
|
||||
data.get("first_author")
|
||||
or data["authors"][0].get("name", "")
|
||||
)
|
||||
cursor = connection.execute(
|
||||
"""
|
||||
INSERT INTO papers(
|
||||
doi, title, title_normalized, first_author, year, journal,
|
||||
volume, issue, pages, publisher, article_type, language,
|
||||
abstract, summary_ja, relevance, priority, review_status,
|
||||
category_id, pdf_path, file_size, file_mtime, file_sha256,
|
||||
source, landing_url, pdf_url, url_source, access_status,
|
||||
url_checked_at, created_at, updated_at, checked_at, notes
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
||||
)
|
||||
""",
|
||||
(
|
||||
normalize_doi(data.get("doi")),
|
||||
data["title"],
|
||||
normalize_text(str(data["title"])),
|
||||
first_author,
|
||||
data.get("year"),
|
||||
data.get("journal"),
|
||||
data.get("volume"),
|
||||
data.get("issue"),
|
||||
data.get("pages"),
|
||||
data.get("publisher"),
|
||||
data.get("article_type"),
|
||||
data.get("language"),
|
||||
data.get("abstract"),
|
||||
data.get("summary_ja"),
|
||||
int(data.get("relevance", 0)),
|
||||
data.get("priority", "D"),
|
||||
data.get("review_status", "unreviewed"),
|
||||
category_id,
|
||||
data.get("pdf_path"),
|
||||
data["file_size"],
|
||||
data["file_mtime"],
|
||||
data["file_sha256"],
|
||||
data.get("source"),
|
||||
data.get("landing_url"),
|
||||
data.get("pdf_url"),
|
||||
data.get("url_source"),
|
||||
data.get("access_status"),
|
||||
data.get("url_checked_at"),
|
||||
now,
|
||||
now,
|
||||
now,
|
||||
data.get("notes"),
|
||||
),
|
||||
)
|
||||
return int(cursor.lastrowid)
|
||||
|
||||
|
||||
def register_authors(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
authors: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""論文著者を登録する。"""
|
||||
existing = connection.execute(
|
||||
"SELECT 1 FROM paper_authors WHERE paper_id = ? LIMIT 1",
|
||||
(paper_id,),
|
||||
).fetchone()
|
||||
if existing:
|
||||
return
|
||||
|
||||
for order, author in enumerate(authors, start=1):
|
||||
author_id = get_author_id(connection, author)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO paper_authors(
|
||||
paper_id, author_id, author_order, is_corresponding
|
||||
) VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
paper_id,
|
||||
author_id,
|
||||
order,
|
||||
int(bool(author.get("is_corresponding", False))),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def register_keywords(
|
||||
connection: sqlite3.Connection,
|
||||
paper_id: int,
|
||||
keywords: list[Any],
|
||||
) -> None:
|
||||
"""論文キーワードを登録する。"""
|
||||
for keyword in keywords:
|
||||
keyword_id = get_keyword_id(connection, keyword)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT OR IGNORE INTO paper_keywords(
|
||||
paper_id, keyword_id
|
||||
) VALUES (?, ?)
|
||||
""",
|
||||
(paper_id, keyword_id),
|
||||
)
|
||||
|
||||
|
||||
def register_one(
|
||||
connection: sqlite3.Connection,
|
||||
json_path: Path,
|
||||
pdf_root: Path,
|
||||
) -> dict[str, Any]:
|
||||
"""1件のJSONを登録する。"""
|
||||
data = load_json(json_path)
|
||||
validate_json(data)
|
||||
add_file_metadata(data, pdf_root)
|
||||
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
existing = find_existing(connection, data)
|
||||
category_id = get_category_id(
|
||||
connection,
|
||||
str(data.get("category", "Other")),
|
||||
)
|
||||
now = now_iso()
|
||||
|
||||
if existing:
|
||||
paper_id = int(existing["id"])
|
||||
update_existing_paper(
|
||||
connection,
|
||||
paper_id,
|
||||
category_id,
|
||||
data,
|
||||
now,
|
||||
)
|
||||
status = "updated"
|
||||
else:
|
||||
paper_id = insert_paper(
|
||||
connection,
|
||||
category_id,
|
||||
data,
|
||||
now,
|
||||
)
|
||||
status = "registered"
|
||||
|
||||
register_authors(connection, paper_id, data["authors"])
|
||||
register_keywords(
|
||||
connection,
|
||||
paper_id,
|
||||
data.get("keywords", []),
|
||||
)
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO processing_log(
|
||||
pdf_path, paper_id, status, message, processed_at
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(data["pdf_path"], paper_id, status, status, now),
|
||||
)
|
||||
|
||||
integrity = connection.execute(
|
||||
"PRAGMA integrity_check"
|
||||
).fetchone()[0]
|
||||
if integrity != "ok":
|
||||
raise RuntimeError(integrity)
|
||||
|
||||
connection.commit()
|
||||
return {
|
||||
"status": status,
|
||||
"paper_id": paper_id,
|
||||
"integrity_check": integrity,
|
||||
}
|
||||
except Exception:
|
||||
connection.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def process_files(args: argparse.Namespace) -> int:
|
||||
"""対象JSONを一括処理する。"""
|
||||
input_dir = Path(args.input_dir)
|
||||
database = Path(args.database)
|
||||
pdf_root = Path(args.pdf_root).expanduser().resolve()
|
||||
files = sorted(input_dir.glob(args.pattern))
|
||||
|
||||
if not files:
|
||||
print("新規JSONはありません。")
|
||||
return 0
|
||||
if not database.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"database not found: {database}"
|
||||
)
|
||||
|
||||
if CONFIG.processing.backup_database:
|
||||
backup = backup_database(database)
|
||||
print(f"Backup: {backup}")
|
||||
|
||||
counts = {"registered": 0, "updated": 0, "failed": 0}
|
||||
with sqlite3.connect(database) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
|
||||
for path in files:
|
||||
print(f"=== {path} ===")
|
||||
try:
|
||||
result = register_one(connection, path, pdf_root)
|
||||
counts[result["status"]] += 1
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
except Exception as exc:
|
||||
counts["failed"] += 1
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
if not args.continue_on_error:
|
||||
return 1
|
||||
|
||||
print()
|
||||
print("=== 一括登録結果 ===")
|
||||
print(f"対象: {len(files)}")
|
||||
print(f"新規登録: {counts['registered']}")
|
||||
print(f"既存更新: {counts['updated']}")
|
||||
print(f"失敗: {counts['failed']}")
|
||||
return 1 if counts["failed"] else 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""メイン処理。"""
|
||||
try:
|
||||
return process_files(parse_args())
|
||||
except Exception as exc:
|
||||
print(f"update_all.py: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+442
@@ -0,0 +1,442 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SHA256が同じPDFの移動・改名をpapers.dbへ反映する。
|
||||
|
||||
PDF内容が変わってSHA256も変わった場合は更新しない。
|
||||
その場合はdelete_paper.pyでDBレコードを削除し、
|
||||
通常の登録フローで再登録する。
|
||||
|
||||
既定はdry-run。実際に更新する場合は--applyを指定する。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from common import CONFIG, resolve_pdf_path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PdfFile:
|
||||
relative_path: str
|
||||
sha256: str
|
||||
file_size: int
|
||||
file_mtime: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PathUpdate:
|
||||
paper_id: int
|
||||
title: str
|
||||
old_path: str
|
||||
new_path: str
|
||||
sha256: str
|
||||
file_size: int
|
||||
file_mtime: float
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="SHA256が同じPDFの移動・改名をDBへ反映する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--database",
|
||||
type=Path,
|
||||
default=CONFIG.path.database,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pdf-root",
|
||||
type=Path,
|
||||
default=CONFIG.path.pdf_root,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scan-all",
|
||||
action="store_true",
|
||||
help="PDF_ROOT全体を走査する",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--apply",
|
||||
action="store_true",
|
||||
help="更新を実際にDBへ反映する",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-backup",
|
||||
action="store_true",
|
||||
help="更新前バックアップを作成しない",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def validate_paths(database: Path, pdf_root: Path) -> None:
|
||||
if not database.is_file():
|
||||
raise FileNotFoundError(database)
|
||||
|
||||
if not pdf_root.is_dir():
|
||||
raise NotADirectoryError(pdf_root)
|
||||
|
||||
|
||||
def scan_roots(
|
||||
pdf_root: Path,
|
||||
scan_all: bool,
|
||||
) -> tuple[Path, ...]:
|
||||
if scan_all:
|
||||
return (pdf_root,)
|
||||
|
||||
roots = tuple(
|
||||
pdf_root / name
|
||||
for name in CONFIG.processing.scan_directories
|
||||
)
|
||||
|
||||
for root in roots:
|
||||
if not root.is_dir():
|
||||
raise NotADirectoryError(root)
|
||||
|
||||
return roots
|
||||
|
||||
|
||||
def iter_pdf_paths(
|
||||
roots: Iterable[Path],
|
||||
) -> Iterable[Path]:
|
||||
seen: set[Path] = set()
|
||||
|
||||
for root in roots:
|
||||
for path in root.rglob("*"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
|
||||
if path.suffix.lower() != ".pdf":
|
||||
continue
|
||||
|
||||
resolved = path.resolve()
|
||||
|
||||
if resolved in seen:
|
||||
continue
|
||||
|
||||
seen.add(resolved)
|
||||
yield resolved
|
||||
|
||||
|
||||
def calculate_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
|
||||
with path.open("rb") as file:
|
||||
for chunk in iter(
|
||||
lambda: file.read(1024 * 1024),
|
||||
b"",
|
||||
):
|
||||
digest.update(chunk)
|
||||
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def inspect_pdf(
|
||||
path: Path,
|
||||
pdf_root: Path,
|
||||
) -> PdfFile:
|
||||
stat = path.stat()
|
||||
|
||||
return PdfFile(
|
||||
relative_path=path.relative_to(
|
||||
pdf_root
|
||||
).as_posix(),
|
||||
sha256=calculate_sha256(path),
|
||||
file_size=stat.st_size,
|
||||
file_mtime=stat.st_mtime,
|
||||
)
|
||||
|
||||
|
||||
def build_sha_index(
|
||||
pdf_root: Path,
|
||||
roots: tuple[Path, ...],
|
||||
) -> dict[str, list[PdfFile]]:
|
||||
index: dict[str, list[PdfFile]] = defaultdict(list)
|
||||
|
||||
paths = sorted(
|
||||
iter_pdf_paths(roots),
|
||||
key=lambda item: str(item).casefold(),
|
||||
)
|
||||
|
||||
for number, path in enumerate(paths, start=1):
|
||||
pdf = inspect_pdf(path, pdf_root)
|
||||
index[pdf.sha256].append(pdf)
|
||||
|
||||
print(
|
||||
f"\rPDFを走査中: {number}件",
|
||||
end="",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if paths:
|
||||
print()
|
||||
|
||||
return dict(index)
|
||||
|
||||
|
||||
def open_database(
|
||||
database: Path,
|
||||
) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
|
||||
def load_papers(
|
||||
connection: sqlite3.Connection,
|
||||
) -> list[sqlite3.Row]:
|
||||
sql = """
|
||||
SELECT id, title, pdf_path, file_sha256
|
||||
FROM papers
|
||||
WHERE file_sha256 IS NOT NULL
|
||||
AND TRIM(file_sha256) <> ''
|
||||
ORDER BY id
|
||||
"""
|
||||
return connection.execute(sql).fetchall()
|
||||
|
||||
|
||||
def current_path_is_valid(
|
||||
pdf_root: Path,
|
||||
row: sqlite3.Row,
|
||||
) -> bool:
|
||||
if not row["pdf_path"]:
|
||||
return False
|
||||
|
||||
try:
|
||||
path = resolve_pdf_path(
|
||||
pdf_root,
|
||||
row["pdf_path"],
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if not path.is_file():
|
||||
return False
|
||||
|
||||
return (
|
||||
calculate_sha256(path)
|
||||
== row["file_sha256"].strip().lower()
|
||||
)
|
||||
|
||||
|
||||
def find_updates(
|
||||
rows: Iterable[sqlite3.Row],
|
||||
index: dict[str, list[PdfFile]],
|
||||
pdf_root: Path,
|
||||
) -> tuple[
|
||||
list[PathUpdate],
|
||||
list[sqlite3.Row],
|
||||
list[sqlite3.Row],
|
||||
]:
|
||||
updates: list[PathUpdate] = []
|
||||
ambiguous: list[sqlite3.Row] = []
|
||||
not_found: list[sqlite3.Row] = []
|
||||
|
||||
for row in rows:
|
||||
if current_path_is_valid(pdf_root, row):
|
||||
continue
|
||||
|
||||
sha256 = row["file_sha256"].strip().lower()
|
||||
candidates = index.get(sha256, [])
|
||||
|
||||
if not candidates:
|
||||
not_found.append(row)
|
||||
continue
|
||||
|
||||
if len(candidates) != 1:
|
||||
ambiguous.append(row)
|
||||
continue
|
||||
|
||||
candidate = candidates[0]
|
||||
|
||||
updates.append(
|
||||
PathUpdate(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
old_path=row["pdf_path"] or "",
|
||||
new_path=candidate.relative_path,
|
||||
sha256=sha256,
|
||||
file_size=candidate.file_size,
|
||||
file_mtime=candidate.file_mtime,
|
||||
)
|
||||
)
|
||||
|
||||
return updates, ambiguous, not_found
|
||||
|
||||
|
||||
def print_updates(
|
||||
updates: Iterable[PathUpdate],
|
||||
) -> None:
|
||||
items = list(updates)
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
print()
|
||||
print("=== 更新候補 ===")
|
||||
|
||||
for item in items:
|
||||
print(f"id: {item.paper_id}")
|
||||
print(f"title: {item.title}")
|
||||
print(f"old: {item.old_path or '(empty)'}")
|
||||
print(f"new: {item.new_path}")
|
||||
print()
|
||||
|
||||
|
||||
def print_rows(
|
||||
title: str,
|
||||
rows: Iterable[sqlite3.Row],
|
||||
) -> None:
|
||||
items = list(rows)
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
print()
|
||||
print(f"=== {title} ===")
|
||||
|
||||
for row in items:
|
||||
print(
|
||||
f"id={row['id']}: "
|
||||
f"{row['pdf_path']} "
|
||||
f"({row['title']})"
|
||||
)
|
||||
|
||||
|
||||
def create_backup(database: Path) -> Path:
|
||||
backup = database.with_name(
|
||||
database.name + ".before-path-update"
|
||||
)
|
||||
|
||||
counter = 1
|
||||
|
||||
while backup.exists():
|
||||
backup = database.with_name(
|
||||
database.name
|
||||
+ f".before-path-update.{counter}"
|
||||
)
|
||||
counter += 1
|
||||
|
||||
shutil.copy2(database, backup)
|
||||
return backup
|
||||
|
||||
|
||||
def apply_updates(
|
||||
connection: sqlite3.Connection,
|
||||
updates: Iterable[PathUpdate],
|
||||
) -> int:
|
||||
sql = """
|
||||
UPDATE papers
|
||||
SET pdf_path = ?,
|
||||
file_size = ?,
|
||||
file_mtime = ?
|
||||
WHERE id = ?
|
||||
AND lower(file_sha256) = ?
|
||||
"""
|
||||
|
||||
count = 0
|
||||
|
||||
with connection:
|
||||
for item in updates:
|
||||
cursor = connection.execute(
|
||||
sql,
|
||||
(
|
||||
item.new_path,
|
||||
item.file_size,
|
||||
item.file_mtime,
|
||||
item.paper_id,
|
||||
item.sha256,
|
||||
),
|
||||
)
|
||||
count += cursor.rowcount
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
try:
|
||||
database = args.database.expanduser().resolve()
|
||||
pdf_root = args.pdf_root.expanduser().resolve()
|
||||
|
||||
validate_paths(database, pdf_root)
|
||||
|
||||
index = build_sha_index(
|
||||
pdf_root,
|
||||
scan_roots(pdf_root, args.scan_all),
|
||||
)
|
||||
|
||||
with open_database(database) as connection:
|
||||
rows = load_papers(connection)
|
||||
updates, ambiguous, not_found = (
|
||||
find_updates(
|
||||
rows,
|
||||
index,
|
||||
pdf_root,
|
||||
)
|
||||
)
|
||||
|
||||
print_updates(updates)
|
||||
print_rows(
|
||||
"同一SHA256の候補が複数",
|
||||
ambiguous,
|
||||
)
|
||||
print_rows(
|
||||
"SHA256一致ファイルなし",
|
||||
not_found,
|
||||
)
|
||||
|
||||
if args.apply and updates:
|
||||
if not args.no_backup:
|
||||
backup = create_backup(database)
|
||||
print(f"DBバックアップ: {backup}")
|
||||
|
||||
changed = apply_updates(
|
||||
connection,
|
||||
updates,
|
||||
)
|
||||
print(f"DB更新: {changed} 件")
|
||||
|
||||
print()
|
||||
print("=== 概要 ===")
|
||||
print(
|
||||
"走査PDF: "
|
||||
f"{sum(len(v) for v in index.values())} 件"
|
||||
)
|
||||
print(f"DB照合対象: {len(rows)} 件")
|
||||
print(f"更新候補: {len(updates)} 件")
|
||||
print(f"曖昧な一致: {len(ambiguous)} 件")
|
||||
print(f"一致なし: {len(not_found)} 件")
|
||||
print(
|
||||
"実行モード: "
|
||||
+ ("更新実行" if args.apply else "dry-run")
|
||||
)
|
||||
|
||||
if updates and not args.apply:
|
||||
print()
|
||||
print(
|
||||
"更新するには --apply を指定してください。"
|
||||
)
|
||||
|
||||
return 0
|
||||
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"update_pdf_paths.py: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+312
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from common import CONFIG, resolve_pdf_path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MissingPdf:
|
||||
"""見つからないPDFの情報。"""
|
||||
|
||||
paper_id: int
|
||||
title: str
|
||||
pdf_path: str
|
||||
resolved_path: str | None
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IntegrityResult:
|
||||
"""SQLite整合性検証の結果。"""
|
||||
|
||||
integrity: str
|
||||
foreign_key_count: int
|
||||
errors: tuple[str, ...]
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""コマンドライン引数を解析する。"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="papers.dbの整合性を検証する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"database",
|
||||
nargs="?",
|
||||
default=str(CONFIG.path.database),
|
||||
help="検証するSQLiteデータベース",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pdf-root",
|
||||
default=str(CONFIG.path.pdf_root),
|
||||
help="PDF_ROOTのパス",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def open_database(database: Path) -> sqlite3.Connection:
|
||||
"""SQLiteデータベースへ接続する。"""
|
||||
connection = sqlite3.connect(database)
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
|
||||
def check_integrity(
|
||||
connection: sqlite3.Connection,
|
||||
) -> IntegrityResult:
|
||||
"""SQLite本体と外部キーの整合性を検証する。"""
|
||||
errors: list[str] = []
|
||||
|
||||
integrity = connection.execute(
|
||||
"PRAGMA integrity_check"
|
||||
).fetchone()[0]
|
||||
|
||||
if integrity != "ok":
|
||||
errors.append(f"integrity_check: {integrity}")
|
||||
|
||||
foreign_keys = connection.execute(
|
||||
"PRAGMA foreign_key_check"
|
||||
).fetchall()
|
||||
|
||||
foreign_key_count = len(foreign_keys)
|
||||
|
||||
if foreign_key_count:
|
||||
errors.append(
|
||||
f"foreign keys: {foreign_key_count}"
|
||||
)
|
||||
|
||||
return IntegrityResult(
|
||||
integrity=integrity,
|
||||
foreign_key_count=foreign_key_count,
|
||||
errors=tuple(errors),
|
||||
)
|
||||
|
||||
|
||||
def validation_queries() -> tuple[tuple[str, str], ...]:
|
||||
"""DB内容の検証用SQLを返す。"""
|
||||
return (
|
||||
(
|
||||
"duplicate DOI",
|
||||
"""
|
||||
SELECT doi
|
||||
FROM papers
|
||||
WHERE doi IS NOT NULL
|
||||
AND doi <> ''
|
||||
GROUP BY doi
|
||||
HAVING COUNT(*) > 1
|
||||
""",
|
||||
),
|
||||
(
|
||||
"duplicate SHA256",
|
||||
"""
|
||||
SELECT file_sha256
|
||||
FROM papers
|
||||
WHERE file_sha256 IS NOT NULL
|
||||
AND file_sha256 <> ''
|
||||
GROUP BY file_sha256
|
||||
HAVING COUNT(*) > 1
|
||||
""",
|
||||
),
|
||||
(
|
||||
"empty title",
|
||||
"""
|
||||
SELECT id
|
||||
FROM papers
|
||||
WHERE title IS NULL
|
||||
OR TRIM(title) = ''
|
||||
""",
|
||||
),
|
||||
(
|
||||
"invalid relevance",
|
||||
"""
|
||||
SELECT id
|
||||
FROM papers
|
||||
WHERE relevance NOT BETWEEN 0 AND 5
|
||||
""",
|
||||
),
|
||||
(
|
||||
"invalid priority",
|
||||
"""
|
||||
SELECT id
|
||||
FROM papers
|
||||
WHERE priority NOT IN ('A', 'B', 'C', 'D')
|
||||
""",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def check_database_values(
|
||||
connection: sqlite3.Connection,
|
||||
) -> list[str]:
|
||||
"""重複や値域を検証する。"""
|
||||
errors: list[str] = []
|
||||
|
||||
for label, sql in validation_queries():
|
||||
rows = connection.execute(sql).fetchall()
|
||||
|
||||
if rows:
|
||||
errors.append(f"{label}: {len(rows)}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def iter_papers_with_pdf(
|
||||
connection: sqlite3.Connection,
|
||||
) -> Iterable[sqlite3.Row]:
|
||||
"""PDFパスが登録された論文を返す。"""
|
||||
sql = """
|
||||
SELECT id, title, pdf_path
|
||||
FROM papers
|
||||
WHERE pdf_path IS NOT NULL
|
||||
AND TRIM(pdf_path) <> ''
|
||||
ORDER BY id
|
||||
"""
|
||||
return connection.execute(sql)
|
||||
|
||||
|
||||
def inspect_pdf(
|
||||
pdf_root: Path,
|
||||
row: sqlite3.Row,
|
||||
) -> MissingPdf | None:
|
||||
"""1件のPDFが存在するか確認する。"""
|
||||
try:
|
||||
path = resolve_pdf_path(
|
||||
pdf_root,
|
||||
row["pdf_path"],
|
||||
)
|
||||
|
||||
if path.is_file():
|
||||
return None
|
||||
|
||||
return MissingPdf(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
pdf_path=row["pdf_path"],
|
||||
resolved_path=str(path),
|
||||
reason="file does not exist",
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
return MissingPdf(
|
||||
paper_id=row["id"],
|
||||
title=row["title"],
|
||||
pdf_path=row["pdf_path"],
|
||||
resolved_path=None,
|
||||
reason=f"{type(exc).__name__}: {exc}",
|
||||
)
|
||||
|
||||
|
||||
def find_missing_pdfs(
|
||||
connection: sqlite3.Connection,
|
||||
pdf_root: Path,
|
||||
) -> list[MissingPdf]:
|
||||
"""存在しないPDFを検索する。"""
|
||||
missing: list[MissingPdf] = []
|
||||
|
||||
for row in iter_papers_with_pdf(connection):
|
||||
item = inspect_pdf(pdf_root, row)
|
||||
|
||||
if item is not None:
|
||||
missing.append(item)
|
||||
|
||||
return missing
|
||||
|
||||
|
||||
def print_missing_pdfs(
|
||||
missing: Iterable[MissingPdf],
|
||||
) -> None:
|
||||
"""見つからないPDFの詳細を表示する。"""
|
||||
items = list(missing)
|
||||
|
||||
if not items:
|
||||
return
|
||||
|
||||
print()
|
||||
print("=== Missing PDFs ===")
|
||||
|
||||
for item in items:
|
||||
print(f"paper_id: {item.paper_id}")
|
||||
print(f"title: {item.title}")
|
||||
print(f"pdf_path: {item.pdf_path}")
|
||||
print(
|
||||
"resolved_path:",
|
||||
item.resolved_path
|
||||
or "(resolve failed)",
|
||||
)
|
||||
print(f"reason: {item.reason}")
|
||||
print()
|
||||
|
||||
|
||||
def print_summary(
|
||||
integrity: IntegrityResult,
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
"""検証結果を表示する。"""
|
||||
print(f"integrity_check: {integrity.integrity}")
|
||||
print(
|
||||
"foreign_key_violations:",
|
||||
integrity.foreign_key_count,
|
||||
)
|
||||
print(f"errors: {len(errors)}")
|
||||
|
||||
for error in errors:
|
||||
print(f"- {error}")
|
||||
|
||||
|
||||
def validate(
|
||||
database: Path,
|
||||
pdf_root: Path,
|
||||
) -> list[str]:
|
||||
"""データベース全体を検証する。"""
|
||||
with open_database(database) as connection:
|
||||
integrity = check_integrity(connection)
|
||||
value_errors = check_database_values(
|
||||
connection
|
||||
)
|
||||
missing = find_missing_pdfs(
|
||||
connection,
|
||||
pdf_root,
|
||||
)
|
||||
|
||||
errors = list(integrity.errors)
|
||||
errors.extend(value_errors)
|
||||
|
||||
if missing:
|
||||
errors.append(
|
||||
f"missing PDF: {len(missing)}"
|
||||
)
|
||||
|
||||
print_missing_pdfs(missing)
|
||||
print_summary(integrity, errors)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
"""メイン処理。"""
|
||||
args = parse_args()
|
||||
|
||||
database = Path(
|
||||
args.database
|
||||
).expanduser().resolve()
|
||||
|
||||
pdf_root = Path(
|
||||
args.pdf_root
|
||||
).expanduser().resolve()
|
||||
|
||||
errors = validate(
|
||||
database,
|
||||
pdf_root,
|
||||
)
|
||||
|
||||
return 1 if errors else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
#+TITLE: 論文データベースプロジェクト
|
||||
|
||||
* 構成
|
||||
|
||||
このディレクトリには、分野固有の設定とデータだけを置く。
|
||||
|
||||
#+begin_example
|
||||
config.py
|
||||
papers.db
|
||||
papers.org
|
||||
papers.publish.db
|
||||
logs/
|
||||
tmp/
|
||||
#+end_example
|
||||
|
||||
=CLAUDE.md= とPythonスクリプトは共通側に置かれるため、
|
||||
このディレクトリには作成しない。
|
||||
|
||||
* 初期設定
|
||||
|
||||
=config.py= の次の項目を編集する。
|
||||
|
||||
- =pdf_root=
|
||||
- =scan_directories=
|
||||
- =domain=
|
||||
- 必要な場合だけ =remote=
|
||||
|
||||
* 実行
|
||||
|
||||
プロジェクトディレクトリへ移動して実行する。
|
||||
|
||||
#+begin_src sh
|
||||
/path/to/paper_db_common/publish.py
|
||||
#+end_src
|
||||
|
||||
公開用DBとPDFも転送する場合は次を使う。
|
||||
|
||||
#+begin_src sh
|
||||
/path/to/paper_db_common/publish.py --publish
|
||||
#+end_src
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from paperdb_config import AIConfig
|
||||
from paperdb_config import Config
|
||||
from paperdb_config import DomainConfig
|
||||
from paperdb_config import PathConfig
|
||||
from paperdb_config import ProcessingConfig
|
||||
from paperdb_config import RemoteConfig
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
CONFIG = Config(
|
||||
# pdfのパスを指定
|
||||
path=PathConfig(
|
||||
project_root=ROOT,
|
||||
pdf_root=Path.home() / "Documents" / "papers",
|
||||
database=ROOT / "papers.db",
|
||||
publish_database=ROOT / "papers.publish.db",
|
||||
tmp_dir=ROOT / "tmp",
|
||||
text_dir=ROOT / "tmp" / "text",
|
||||
log_dir=ROOT / "logs",
|
||||
papers_org=ROOT / "papers.org",
|
||||
),
|
||||
# scan_directoriesにpdfを置いているdirectryをlist
|
||||
processing=ProcessingConfig(
|
||||
scan_directories=("user1", "user2",),
|
||||
ignore_directories=("old", "tmp", "trash"),
|
||||
max_papers=3,
|
||||
initial_pages=2,
|
||||
fallback_pages=5,
|
||||
min_text_length=1500,
|
||||
update_database=True,
|
||||
validate_database=True,
|
||||
export_org=True,
|
||||
backup_database=True,
|
||||
),
|
||||
# categoriesに分類をlist, 分野名にわかりやすいname
|
||||
domain=DomainConfig(
|
||||
name="分野名",
|
||||
description="収集対象とする論文分野の説明",
|
||||
categories=(
|
||||
"Review",
|
||||
"Other",
|
||||
),
|
||||
default_category="Other",
|
||||
summary_length_ja="100〜200字程度",
|
||||
summary_focus=(
|
||||
"研究目的",
|
||||
"手法",
|
||||
"主な結果",
|
||||
"新規性",
|
||||
"対象分野との関連",
|
||||
),
|
||||
relevance_criteria=(
|
||||
"対象分野と無関係",
|
||||
"わずかに関連",
|
||||
"周辺分野",
|
||||
"関連する基礎研究",
|
||||
"対象分野へ直接応用可能",
|
||||
"対象分野そのもの",
|
||||
),
|
||||
),
|
||||
remote=RemoteConfig(
|
||||
host="user@example.org",
|
||||
database_dir="/home/user/public_html/db",
|
||||
pdf_dir="/home/user/public_html/pdf",
|
||||
),
|
||||
ai=AIConfig(
|
||||
backend="claude",
|
||||
claude_permission_mode="dontAsk",
|
||||
claude_allowed_tools=(
|
||||
"Read",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"Write",
|
||||
"Edit",
|
||||
"WebSearch",
|
||||
"WebFetch",
|
||||
),
|
||||
codex_model=None,
|
||||
codex_sandbox="workspace-write",
|
||||
codex_full_auto=True,
|
||||
),
|
||||
)
|
||||
Reference in New Issue
Block a user