#!/usr/bin/env python3 from __future__ import annotations import hashlib import html import re import unicodedata from datetime import datetime, timezone from pathlib import Path from typing import Any import sys COMMON_ROOT = Path(__file__).resolve().parents[1] if str(COMMON_ROOT) not in sys.path: sys.path.insert(0, str(COMMON_ROOT)) from project_loader import load_config # noqa: E402 _, CONFIG = load_config() ALLOWED_PRIORITIES = set(CONFIG.domain.priorities) ALLOWED_CATEGORIES = set(CONFIG.domain.categories) def now_iso() -> str: return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") def normalize_doi(value: Any) -> str | None: if value is None: return None doi = str(value).strip().lower() if not doi or doi == "unknown": return None prefixes = ( "https://doi.org/", "http://doi.org/", "http://dx.doi.org/", "doi:", ) for prefix in prefixes: if doi.startswith(prefix): doi = doi[len(prefix):] break doi = doi.strip().rstrip(".,;:)]}") return doi or None def normalize_text(value: str) -> str: value = html.unescape(value) value = unicodedata.normalize("NFKC", value) value = value.lower() value = re.sub(r"[\r\n\t]+", " ", value) value = re.sub(r"[^\w\s]", " ", value, flags=re.UNICODE) value = re.sub(r"\s+", " ", value).strip() return value def sha256_file(path: Path) -> str: h = hashlib.sha256() with path.open("rb") as f: for block in iter(lambda: f.read(1024 * 1024), b""): h.update(block) return h.hexdigest() def resolve_pdf_path(pdf_root: Path, stored_path: str) -> Path: path = Path(stored_path) if path.is_absolute(): raise ValueError("pdf_path must be relative to PDF_ROOT") resolved = (pdf_root / path).resolve() root = pdf_root.resolve() try: resolved.relative_to(root) except ValueError as exc: raise ValueError("pdf_path escapes PDF_ROOT") from exc return resolved