Initial commit

paper_db_common/scripts/common.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import hashlib
+import html
+import re
+import unicodedata
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import sys
+
+COMMON_ROOT = Path(__file__).resolve().parents[1]
+if str(COMMON_ROOT) not in sys.path:
+    sys.path.insert(0, str(COMMON_ROOT))
+
+from project_loader import load_config  # noqa: E402
+
+_, CONFIG = load_config()
+ALLOWED_PRIORITIES = set(CONFIG.domain.priorities)
+ALLOWED_CATEGORIES = set(CONFIG.domain.categories)
+def now_iso() -> str:
+    return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
+
+def normalize_doi(value: Any) -> str | None:
+    if value is None:
+        return None
+    doi = str(value).strip().lower()
+    if not doi or doi == "unknown":
+        return None
+    prefixes = (
+        "https://doi.org/",
+        "http://doi.org/",
+        "http://dx.doi.org/",
+        "doi:",
+    )
+    for prefix in prefixes:
+        if doi.startswith(prefix):
+            doi = doi[len(prefix):]
+            break
+    doi = doi.strip().rstrip(".,;:)]}")
+    return doi or None
+
+def normalize_text(value: str) -> str:
+    value = html.unescape(value)
+    value = unicodedata.normalize("NFKC", value)
+    value = value.lower()
+    value = re.sub(r"[\r\n\t]+", " ", value)
+    value = re.sub(r"[^\w\s]", " ", value, flags=re.UNICODE)
+    value = re.sub(r"\s+", " ", value).strip()
+    return value
+
+def sha256_file(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for block in iter(lambda: f.read(1024 * 1024), b""):
+            h.update(block)
+    return h.hexdigest()
+
+def resolve_pdf_path(pdf_root: Path, stored_path: str) -> Path:
+    path = Path(stored_path)
+    if path.is_absolute():
+        raise ValueError("pdf_path must be relative to PDF_ROOT")
+    resolved = (pdf_root / path).resolve()
+    root = pdf_root.resolve()
+    try:
+        resolved.relative_to(root)
+    except ValueError as exc:
+        raise ValueError("pdf_path escapes PDF_ROOT") from exc
+    return resolved