#!/usr/bin/env python3 from __future__ import annotations import argparse import os import shlex import shutil import sqlite3 import subprocess import sys from dataclasses import dataclass from pathlib import Path from typing import Sequence from project_loader import load_config COMMON_ROOT = Path(__file__).resolve().parent SCRIPTS_DIR = COMMON_ROOT / "scripts" @dataclass(frozen=True) class RuntimeOptions: mode: str backend: str max_papers: int def positive_int(value: str) -> int: number = int(value) if number < 1: raise argparse.ArgumentTypeError("1以上を指定してください") return number def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="論文解析、DB整理、公開処理を実行する" ) parser.add_argument( "mode", choices=("process", "sync", "publish", "all"), help="実行モード", ) parser.add_argument( "--config", help="省略時はカレントディレクトリのconfig.py", ) parser.add_argument( "--backend", choices=("claude", "codex"), help=( "使用するAIバックエンド。省略時は" "config.pyの設定を使用" ), ) parser.add_argument( "--max-papers", type=positive_int, metavar="N", help=( "1回に処理する最大論文数。省略時は" "config.pyの設定を使用" ), ) return parser.parse_args() def run_command( command: Sequence[object], *, config_path: Path, config: object, check: bool = True, ) -> subprocess.CompletedProcess[str]: args = [str(item) for item in command] print("+", " ".join(args)) env = os.environ.copy() env["PAPER_DB_CONFIG"] = str(config_path) return subprocess.run( args, cwd=config.path.project_root, env=env, text=True, check=check, ) def require_commands(names: Sequence[str]) -> None: for name in names: if shutil.which(name) is None: raise RuntimeError( f"必要なコマンドが見つかりません: {name}" ) def script(name: str) -> Path: path = SCRIPTS_DIR / name if not path.is_file(): raise FileNotFoundError(path) return path def ensure_database(config_path: Path, config: object) -> None: if config.path.database.is_file(): return print("=== データベース初期化 ===") run_command( [sys.executable, script("init_db.py"), config.path.database], config_path=config_path, config=config, ) def prepare_work_directories(config: object) -> None: shutil.rmtree(config.path.tmp_dir, ignore_errors=True) config.path.text_dir.mkdir(parents=True, exist_ok=True) config.path.log_dir.mkdir(parents=True, exist_ok=True) def extract_text( config_path: Path, config: object, max_papers: int ) -> bool: p = config.processing command: list[object] = [ sys.executable, script("extract_text.py"), "--database", config.path.database, "--pdf-root", config.path.pdf_root, "--output-dir", config.path.text_dir, "--limit", max_papers, "--pages", p.initial_pages, "--fallback-pages", p.fallback_pages, "--min-text-length", p.min_text_length, ] for directory in p.scan_directories: command.extend(["--scan-dir", directory]) result = run_command( command, config_path=config_path, config=config, check=False, ) if result.returncode == 0: return True if result.returncode == 10: print("=== 未登録PDFはありません ===") return False raise RuntimeError( f"extract_text.pyが失敗しました: {result.returncode}" ) def process_new_papers( config_path: Path, config: object, options: RuntimeOptions ) -> None: config.validate() require_commands(("pdftotext", options.backend)) ensure_database(config_path, config) prepare_work_directories(config) if not extract_text(config_path, config, options.max_papers): return run_command( [sys.executable, script("run_ai.py"), "--backend", options.backend, "--max-papers", options.max_papers], config_path=config_path, config=config, ) if config.processing.update_database: run_command( [sys.executable, script("update_all.py"), "--input-dir", config.path.tmp_dir, "--database", config.path.database, "--pdf-root", config.path.pdf_root, "--continue-on-error"], config_path=config_path, config=config, ) def sync_local_database(config_path: Path, config: object) -> None: """PDFパス更新、DB検証、Org出力を実行する。""" config.validate() ensure_database(config_path, config) run_command( [ sys.executable, script("update_pdf_paths.py"), "--database", config.path.database, "--pdf-root", config.path.pdf_root, "--apply", ], config_path=config_path, config=config, ) if config.processing.validate_database: run_command( [ sys.executable, script("validate_db.py"), config.path.database, "--pdf-root", config.path.pdf_root, ], config_path=config_path, config=config, ) if config.processing.export_org: run_command( [ sys.executable, script("export_org.py"), config.path.database, "-o", config.path.papers_org, ], config_path=config_path, config=config, ) def publish_remote(config_path: Path, config: object) -> None: """PDFと公開用DBをリモートへ反映する。""" config.validate(require_remote=True) require_commands(("rsync", "scp", "ssh")) if not config.path.database.is_file(): raise FileNotFoundError(config.path.database) sync_pdfs(config_path, config) publish_database(config_path, config) def create_publish_database(config: object) -> None: config.path.publish_database.unlink(missing_ok=True) with sqlite3.connect(config.path.database) as source: with sqlite3.connect(config.path.publish_database) as target: source.backup(target) with sqlite3.connect(config.path.publish_database) as con: con.execute("PRAGMA journal_mode=DELETE") con.commit() def shell_quote(value: str) -> str: return shlex.quote(value) def publish_database(config_path: Path, config: object) -> None: config.validate(require_remote=True) require_commands(("scp", "ssh")) if not config.path.database.is_file(): raise FileNotFoundError(config.path.database) create_publish_database(config) destination = ( f"{config.remote.host}:" f"{config.remote.database_dir}/papers.db.new" ) run_command( ["scp", config.path.publish_database, destination], config_path=config_path, config=config, ) remote_dir = shell_quote(config.remote.database_dir) remote_script = f"""set -eu cd {remote_dir} sqlite3 papers.db.new 'PRAGMA integrity_check;' mv -f papers.db.new papers.db sqlite3 papers.db 'PRAGMA integrity_check;' """ run_command( ["ssh", config.remote.host, remote_script], config_path=config_path, config=config, ) def sync_pdfs(config_path: Path, config: object) -> None: config.validate(require_remote=True) require_commands(("rsync",)) run_command( ["rsync", "-az", f"{config.path.pdf_root}/", f"{config.remote.host}:{config.remote.pdf_dir}/"], config_path=config_path, config=config, ) def main() -> int: args = parse_args() try: config_path, config = load_config(args.config) options = RuntimeOptions( mode=args.mode, backend=args.backend or config.ai.backend, max_papers=( args.max_papers or config.processing.max_papers ), ) if options.mode in {"process", "all"}: process_new_papers(config_path, config, options) if options.mode in {"sync", "all"}: sync_local_database(config_path, config) if options.mode in {"publish", "all"}: publish_remote(config_path, config) print("=== 完了 ===") return 0 except KeyboardInterrupt: print("中断しました。", file=sys.stderr) return 130 except Exception as exc: print(f"publish.py: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())