72 lines
2.0 KiB
Python
Executable File
72 lines
2.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import html
|
|
import re
|
|
import unicodedata
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import sys
|
|
|
|
COMMON_ROOT = Path(__file__).resolve().parents[1]
|
|
if str(COMMON_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(COMMON_ROOT))
|
|
|
|
from project_loader import load_config # noqa: E402
|
|
|
|
_, CONFIG = load_config()
|
|
ALLOWED_PRIORITIES = set(CONFIG.domain.priorities)
|
|
ALLOWED_CATEGORIES = set(CONFIG.domain.categories)
|
|
def now_iso() -> str:
|
|
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
|
|
|
def normalize_doi(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
doi = str(value).strip().lower()
|
|
if not doi or doi == "unknown":
|
|
return None
|
|
prefixes = (
|
|
"https://doi.org/",
|
|
"http://doi.org/",
|
|
"http://dx.doi.org/",
|
|
"doi:",
|
|
)
|
|
for prefix in prefixes:
|
|
if doi.startswith(prefix):
|
|
doi = doi[len(prefix):]
|
|
break
|
|
doi = doi.strip().rstrip(".,;:)]}")
|
|
return doi or None
|
|
|
|
def normalize_text(value: str) -> str:
|
|
value = html.unescape(value)
|
|
value = unicodedata.normalize("NFKC", value)
|
|
value = value.lower()
|
|
value = re.sub(r"[\r\n\t]+", " ", value)
|
|
value = re.sub(r"[^\w\s]", " ", value, flags=re.UNICODE)
|
|
value = re.sub(r"\s+", " ", value).strip()
|
|
return value
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as f:
|
|
for block in iter(lambda: f.read(1024 * 1024), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
def resolve_pdf_path(pdf_root: Path, stored_path: str) -> Path:
|
|
path = Path(stored_path)
|
|
if path.is_absolute():
|
|
raise ValueError("pdf_path must be relative to PDF_ROOT")
|
|
resolved = (pdf_root / path).resolve()
|
|
root = pdf_root.resolve()
|
|
try:
|
|
resolved.relative_to(root)
|
|
except ValueError as exc:
|
|
raise ValueError("pdf_path escapes PDF_ROOT") from exc
|
|
return resolved
|