Initial commit
This commit is contained in:
Executable
+323
@@ -0,0 +1,323 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Sequence
|
||||
|
||||
from project_loader import load_config
|
||||
|
||||
COMMON_ROOT = Path(__file__).resolve().parent
|
||||
SCRIPTS_DIR = COMMON_ROOT / "scripts"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeOptions:
|
||||
mode: str
|
||||
backend: str
|
||||
max_papers: int
|
||||
|
||||
|
||||
def positive_int(value: str) -> int:
|
||||
number = int(value)
|
||||
if number < 1:
|
||||
raise argparse.ArgumentTypeError("1以上を指定してください")
|
||||
return number
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="論文解析、DB整理、公開処理を実行する"
|
||||
)
|
||||
parser.add_argument(
|
||||
"mode",
|
||||
choices=("process", "sync", "publish", "all"),
|
||||
help="実行モード",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config",
|
||||
help="省略時はカレントディレクトリのconfig.py",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
choices=("claude", "codex"),
|
||||
help=(
|
||||
"使用するAIバックエンド。省略時は"
|
||||
"config.pyの設定を使用"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-papers",
|
||||
type=positive_int,
|
||||
metavar="N",
|
||||
help=(
|
||||
"1回に処理する最大論文数。省略時は"
|
||||
"config.pyの設定を使用"
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def run_command(
|
||||
command: Sequence[object],
|
||||
*,
|
||||
config_path: Path,
|
||||
config: object,
|
||||
check: bool = True,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
args = [str(item) for item in command]
|
||||
print("+", " ".join(args))
|
||||
env = os.environ.copy()
|
||||
env["PAPER_DB_CONFIG"] = str(config_path)
|
||||
return subprocess.run(
|
||||
args,
|
||||
cwd=config.path.project_root,
|
||||
env=env,
|
||||
text=True,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def require_commands(names: Sequence[str]) -> None:
|
||||
for name in names:
|
||||
if shutil.which(name) is None:
|
||||
raise RuntimeError(
|
||||
f"必要なコマンドが見つかりません: {name}"
|
||||
)
|
||||
|
||||
|
||||
def script(name: str) -> Path:
|
||||
path = SCRIPTS_DIR / name
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
return path
|
||||
|
||||
|
||||
def ensure_database(config_path: Path, config: object) -> None:
|
||||
if config.path.database.is_file():
|
||||
return
|
||||
print("=== データベース初期化 ===")
|
||||
run_command(
|
||||
[sys.executable, script("init_db.py"), config.path.database],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def prepare_work_directories(config: object) -> None:
|
||||
shutil.rmtree(config.path.tmp_dir, ignore_errors=True)
|
||||
config.path.text_dir.mkdir(parents=True, exist_ok=True)
|
||||
config.path.log_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def extract_text(
|
||||
config_path: Path, config: object, max_papers: int
|
||||
) -> bool:
|
||||
p = config.processing
|
||||
command: list[object] = [
|
||||
sys.executable, script("extract_text.py"),
|
||||
"--database", config.path.database,
|
||||
"--pdf-root", config.path.pdf_root,
|
||||
"--output-dir", config.path.text_dir,
|
||||
"--limit", max_papers,
|
||||
"--pages", p.initial_pages,
|
||||
"--fallback-pages", p.fallback_pages,
|
||||
"--min-text-length", p.min_text_length,
|
||||
]
|
||||
for directory in p.scan_directories:
|
||||
command.extend(["--scan-dir", directory])
|
||||
result = run_command(
|
||||
command,
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
return True
|
||||
if result.returncode == 10:
|
||||
print("=== 未登録PDFはありません ===")
|
||||
return False
|
||||
raise RuntimeError(
|
||||
f"extract_text.pyが失敗しました: {result.returncode}"
|
||||
)
|
||||
|
||||
|
||||
def process_new_papers(
|
||||
config_path: Path, config: object, options: RuntimeOptions
|
||||
) -> None:
|
||||
config.validate()
|
||||
require_commands(("pdftotext", options.backend))
|
||||
ensure_database(config_path, config)
|
||||
prepare_work_directories(config)
|
||||
if not extract_text(config_path, config, options.max_papers):
|
||||
return
|
||||
run_command(
|
||||
[sys.executable, script("run_ai.py"),
|
||||
"--backend", options.backend,
|
||||
"--max-papers", options.max_papers],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
if config.processing.update_database:
|
||||
run_command(
|
||||
[sys.executable, script("update_all.py"),
|
||||
"--input-dir", config.path.tmp_dir,
|
||||
"--database", config.path.database,
|
||||
"--pdf-root", config.path.pdf_root,
|
||||
"--continue-on-error"],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def sync_local_database(config_path: Path, config: object) -> None:
|
||||
"""PDFパス更新、DB検証、Org出力を実行する。"""
|
||||
config.validate()
|
||||
ensure_database(config_path, config)
|
||||
|
||||
run_command(
|
||||
[
|
||||
sys.executable,
|
||||
script("update_pdf_paths.py"),
|
||||
"--database",
|
||||
config.path.database,
|
||||
"--pdf-root",
|
||||
config.path.pdf_root,
|
||||
"--apply",
|
||||
],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
if config.processing.validate_database:
|
||||
run_command(
|
||||
[
|
||||
sys.executable,
|
||||
script("validate_db.py"),
|
||||
config.path.database,
|
||||
"--pdf-root",
|
||||
config.path.pdf_root,
|
||||
],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
if config.processing.export_org:
|
||||
run_command(
|
||||
[
|
||||
sys.executable,
|
||||
script("export_org.py"),
|
||||
config.path.database,
|
||||
"-o",
|
||||
config.path.papers_org,
|
||||
],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def publish_remote(config_path: Path, config: object) -> None:
|
||||
"""PDFと公開用DBをリモートへ反映する。"""
|
||||
config.validate(require_remote=True)
|
||||
require_commands(("rsync", "scp", "ssh"))
|
||||
if not config.path.database.is_file():
|
||||
raise FileNotFoundError(config.path.database)
|
||||
sync_pdfs(config_path, config)
|
||||
publish_database(config_path, config)
|
||||
|
||||
|
||||
def create_publish_database(config: object) -> None:
|
||||
config.path.publish_database.unlink(missing_ok=True)
|
||||
with sqlite3.connect(config.path.database) as source:
|
||||
with sqlite3.connect(config.path.publish_database) as target:
|
||||
source.backup(target)
|
||||
with sqlite3.connect(config.path.publish_database) as con:
|
||||
con.execute("PRAGMA journal_mode=DELETE")
|
||||
con.commit()
|
||||
|
||||
|
||||
def shell_quote(value: str) -> str:
|
||||
return shlex.quote(value)
|
||||
|
||||
|
||||
def publish_database(config_path: Path, config: object) -> None:
|
||||
config.validate(require_remote=True)
|
||||
require_commands(("scp", "ssh"))
|
||||
if not config.path.database.is_file():
|
||||
raise FileNotFoundError(config.path.database)
|
||||
create_publish_database(config)
|
||||
destination = (
|
||||
f"{config.remote.host}:"
|
||||
f"{config.remote.database_dir}/papers.db.new"
|
||||
)
|
||||
run_command(
|
||||
["scp", config.path.publish_database, destination],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
remote_dir = shell_quote(config.remote.database_dir)
|
||||
remote_script = f"""set -eu
|
||||
cd {remote_dir}
|
||||
sqlite3 papers.db.new 'PRAGMA integrity_check;'
|
||||
mv -f papers.db.new papers.db
|
||||
sqlite3 papers.db 'PRAGMA integrity_check;'
|
||||
"""
|
||||
run_command(
|
||||
["ssh", config.remote.host, remote_script],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def sync_pdfs(config_path: Path, config: object) -> None:
|
||||
config.validate(require_remote=True)
|
||||
require_commands(("rsync",))
|
||||
run_command(
|
||||
["rsync", "-az", f"{config.path.pdf_root}/",
|
||||
f"{config.remote.host}:{config.remote.pdf_dir}/"],
|
||||
config_path=config_path,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
config_path, config = load_config(args.config)
|
||||
options = RuntimeOptions(
|
||||
mode=args.mode,
|
||||
backend=args.backend or config.ai.backend,
|
||||
max_papers=(
|
||||
args.max_papers
|
||||
or config.processing.max_papers
|
||||
),
|
||||
)
|
||||
|
||||
if options.mode in {"process", "all"}:
|
||||
process_new_papers(config_path, config, options)
|
||||
|
||||
if options.mode in {"sync", "all"}:
|
||||
sync_local_database(config_path, config)
|
||||
|
||||
if options.mode in {"publish", "all"}:
|
||||
publish_remote(config_path, config)
|
||||
|
||||
print("=== 完了 ===")
|
||||
return 0
|
||||
except KeyboardInterrupt:
|
||||
print("中断しました。", file=sys.stderr)
|
||||
return 130
|
||||
except Exception as exc:
|
||||
print(f"publish.py: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user