324 lines
8.9 KiB
Python
Executable File
324 lines
8.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import shlex
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Sequence
|
|
|
|
from project_loader import load_config
|
|
|
|
COMMON_ROOT = Path(__file__).resolve().parent
|
|
SCRIPTS_DIR = COMMON_ROOT / "scripts"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RuntimeOptions:
|
|
mode: str
|
|
backend: str
|
|
max_papers: int
|
|
|
|
|
|
def positive_int(value: str) -> int:
|
|
number = int(value)
|
|
if number < 1:
|
|
raise argparse.ArgumentTypeError("1以上を指定してください")
|
|
return number
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="論文解析、DB整理、公開処理を実行する"
|
|
)
|
|
parser.add_argument(
|
|
"mode",
|
|
choices=("process", "sync", "publish", "all"),
|
|
help="実行モード",
|
|
)
|
|
parser.add_argument(
|
|
"--config",
|
|
help="省略時はカレントディレクトリのconfig.py",
|
|
)
|
|
parser.add_argument(
|
|
"--backend",
|
|
choices=("claude", "codex"),
|
|
help=(
|
|
"使用するAIバックエンド。省略時は"
|
|
"config.pyの設定を使用"
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--max-papers",
|
|
type=positive_int,
|
|
metavar="N",
|
|
help=(
|
|
"1回に処理する最大論文数。省略時は"
|
|
"config.pyの設定を使用"
|
|
),
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def run_command(
|
|
command: Sequence[object],
|
|
*,
|
|
config_path: Path,
|
|
config: object,
|
|
check: bool = True,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
args = [str(item) for item in command]
|
|
print("+", " ".join(args))
|
|
env = os.environ.copy()
|
|
env["PAPER_DB_CONFIG"] = str(config_path)
|
|
return subprocess.run(
|
|
args,
|
|
cwd=config.path.project_root,
|
|
env=env,
|
|
text=True,
|
|
check=check,
|
|
)
|
|
|
|
|
|
def require_commands(names: Sequence[str]) -> None:
|
|
for name in names:
|
|
if shutil.which(name) is None:
|
|
raise RuntimeError(
|
|
f"必要なコマンドが見つかりません: {name}"
|
|
)
|
|
|
|
|
|
def script(name: str) -> Path:
|
|
path = SCRIPTS_DIR / name
|
|
if not path.is_file():
|
|
raise FileNotFoundError(path)
|
|
return path
|
|
|
|
|
|
def ensure_database(config_path: Path, config: object) -> None:
|
|
if config.path.database.is_file():
|
|
return
|
|
print("=== データベース初期化 ===")
|
|
run_command(
|
|
[sys.executable, script("init_db.py"), config.path.database],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
|
|
|
|
def prepare_work_directories(config: object) -> None:
|
|
shutil.rmtree(config.path.tmp_dir, ignore_errors=True)
|
|
config.path.text_dir.mkdir(parents=True, exist_ok=True)
|
|
config.path.log_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def extract_text(
|
|
config_path: Path, config: object, max_papers: int
|
|
) -> bool:
|
|
p = config.processing
|
|
command: list[object] = [
|
|
sys.executable, script("extract_text.py"),
|
|
"--database", config.path.database,
|
|
"--pdf-root", config.path.pdf_root,
|
|
"--output-dir", config.path.text_dir,
|
|
"--limit", max_papers,
|
|
"--pages", p.initial_pages,
|
|
"--fallback-pages", p.fallback_pages,
|
|
"--min-text-length", p.min_text_length,
|
|
]
|
|
for directory in p.scan_directories:
|
|
command.extend(["--scan-dir", directory])
|
|
result = run_command(
|
|
command,
|
|
config_path=config_path,
|
|
config=config,
|
|
check=False,
|
|
)
|
|
if result.returncode == 0:
|
|
return True
|
|
if result.returncode == 10:
|
|
print("=== 未登録PDFはありません ===")
|
|
return False
|
|
raise RuntimeError(
|
|
f"extract_text.pyが失敗しました: {result.returncode}"
|
|
)
|
|
|
|
|
|
def process_new_papers(
|
|
config_path: Path, config: object, options: RuntimeOptions
|
|
) -> None:
|
|
config.validate()
|
|
require_commands(("pdftotext", options.backend))
|
|
ensure_database(config_path, config)
|
|
prepare_work_directories(config)
|
|
if not extract_text(config_path, config, options.max_papers):
|
|
return
|
|
run_command(
|
|
[sys.executable, script("run_ai.py"),
|
|
"--backend", options.backend,
|
|
"--max-papers", options.max_papers],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
if config.processing.update_database:
|
|
run_command(
|
|
[sys.executable, script("update_all.py"),
|
|
"--input-dir", config.path.tmp_dir,
|
|
"--database", config.path.database,
|
|
"--pdf-root", config.path.pdf_root,
|
|
"--continue-on-error"],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
|
|
|
|
def sync_local_database(config_path: Path, config: object) -> None:
|
|
"""PDFパス更新、DB検証、Org出力を実行する。"""
|
|
config.validate()
|
|
ensure_database(config_path, config)
|
|
|
|
run_command(
|
|
[
|
|
sys.executable,
|
|
script("update_pdf_paths.py"),
|
|
"--database",
|
|
config.path.database,
|
|
"--pdf-root",
|
|
config.path.pdf_root,
|
|
"--apply",
|
|
],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
|
|
if config.processing.validate_database:
|
|
run_command(
|
|
[
|
|
sys.executable,
|
|
script("validate_db.py"),
|
|
config.path.database,
|
|
"--pdf-root",
|
|
config.path.pdf_root,
|
|
],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
|
|
if config.processing.export_org:
|
|
run_command(
|
|
[
|
|
sys.executable,
|
|
script("export_org.py"),
|
|
config.path.database,
|
|
"-o",
|
|
config.path.papers_org,
|
|
],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
|
|
|
|
def publish_remote(config_path: Path, config: object) -> None:
|
|
"""PDFと公開用DBをリモートへ反映する。"""
|
|
config.validate(require_remote=True)
|
|
require_commands(("rsync", "scp", "ssh"))
|
|
if not config.path.database.is_file():
|
|
raise FileNotFoundError(config.path.database)
|
|
sync_pdfs(config_path, config)
|
|
publish_database(config_path, config)
|
|
|
|
|
|
def create_publish_database(config: object) -> None:
|
|
config.path.publish_database.unlink(missing_ok=True)
|
|
with sqlite3.connect(config.path.database) as source:
|
|
with sqlite3.connect(config.path.publish_database) as target:
|
|
source.backup(target)
|
|
with sqlite3.connect(config.path.publish_database) as con:
|
|
con.execute("PRAGMA journal_mode=DELETE")
|
|
con.commit()
|
|
|
|
|
|
def shell_quote(value: str) -> str:
|
|
return shlex.quote(value)
|
|
|
|
|
|
def publish_database(config_path: Path, config: object) -> None:
|
|
config.validate(require_remote=True)
|
|
require_commands(("scp", "ssh"))
|
|
if not config.path.database.is_file():
|
|
raise FileNotFoundError(config.path.database)
|
|
create_publish_database(config)
|
|
destination = (
|
|
f"{config.remote.host}:"
|
|
f"{config.remote.database_dir}/papers.db.new"
|
|
)
|
|
run_command(
|
|
["scp", config.path.publish_database, destination],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
remote_dir = shell_quote(config.remote.database_dir)
|
|
remote_script = f"""set -eu
|
|
cd {remote_dir}
|
|
sqlite3 papers.db.new 'PRAGMA integrity_check;'
|
|
mv -f papers.db.new papers.db
|
|
sqlite3 papers.db 'PRAGMA integrity_check;'
|
|
"""
|
|
run_command(
|
|
["ssh", config.remote.host, remote_script],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
|
|
|
|
def sync_pdfs(config_path: Path, config: object) -> None:
|
|
config.validate(require_remote=True)
|
|
require_commands(("rsync",))
|
|
run_command(
|
|
["rsync", "-az", f"{config.path.pdf_root}/",
|
|
f"{config.remote.host}:{config.remote.pdf_dir}/"],
|
|
config_path=config_path,
|
|
config=config,
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
config_path, config = load_config(args.config)
|
|
options = RuntimeOptions(
|
|
mode=args.mode,
|
|
backend=args.backend or config.ai.backend,
|
|
max_papers=(
|
|
args.max_papers
|
|
or config.processing.max_papers
|
|
),
|
|
)
|
|
|
|
if options.mode in {"process", "all"}:
|
|
process_new_papers(config_path, config, options)
|
|
|
|
if options.mode in {"sync", "all"}:
|
|
sync_local_database(config_path, config)
|
|
|
|
if options.mode in {"publish", "all"}:
|
|
publish_remote(config_path, config)
|
|
|
|
print("=== 完了 ===")
|
|
return 0
|
|
except KeyboardInterrupt:
|
|
print("中断しました。", file=sys.stderr)
|
|
return 130
|
|
except Exception as exc:
|
|
print(f"publish.py: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|