diff --git a/B04_PreProcess/B04_PreProcess_Engine.py b/B04_PreProcess/B04_PreProcess_Engine.py index 613cd2743..53f567821 100644 --- a/B04_PreProcess/B04_PreProcess_Engine.py +++ b/B04_PreProcess/B04_PreProcess_Engine.py @@ -22,6 +22,8 @@ from B04_PreProcess.B04_PreProcess_Engine_Ground import ( from B04_PreProcess.B04_PreProcess_Engine_Pipeline import build_all_terrain_models from B04_PreProcess.B04_PreProcess_Engine_Structurize import structurize_las from common_util.common_util_atomic import atomic_write_npz +from common_util.common_util_derived_store import file_stamp +from common_util.common_util_derived_store import key_matches as _key_matches from common_util.common_util_json import atomic_write_json from config.config_system import ( SHEET_SURFACE_AUTO_METHODS, @@ -128,23 +130,6 @@ def _structured_stats(structured: Any, structured_path: Path) -> tuple[dict[str, return stats, xyz -def file_stamp(path: Path | None) -> str | None: - """결과 파일 지문 = 수정시각 ns : 크기. 없으면 None.""" - try: - stat = path.stat() if path is not None else None - except OSError: - return None - return f"{stat.st_mtime_ns}:{stat.st_size}" if stat else None - - -def _key_matches(key_path: Path, key: Any) -> bool: - """결과 옆 열쇠 파일이 지금 입력과 같은지.""" - try: - return json.loads(key_path.read_text(encoding="utf-8")) == key - except (OSError, ValueError): - return False - - def _relative_to_project(project_root: Path, path: Path) -> str: """프로젝트 루트 기준 posix 상대 경로 문자열.""" return path.relative_to(project_root).as_posix() diff --git a/B04_PreProcess/B04_PreProcess_Engine_SheetSurface.py b/B04_PreProcess/B04_PreProcess_Engine_SheetSurface.py index 22ae99d4f..9b6335888 100644 --- a/B04_PreProcess/B04_PreProcess_Engine_SheetSurface.py +++ b/B04_PreProcess/B04_PreProcess_Engine_SheetSurface.py @@ -25,7 +25,6 @@ from typing import Any import numpy as np from pyproj import Transformer -from B04_PreProcess.B04_PreProcess_Engine import file_stamp from B04_PreProcess.B04_PreProcess_Engine_ModelContext import ( atomic_npz, clip_and_compact_mesh, @@ -38,6 +37,7 @@ from B04_PreProcess.B04_PreProcess_Engine_SheetMethods import ( SHEET_METHOD_LABELS, ) from B04_PreProcess.B04_PreProcess_Engine_Watershed_Grid import grid_spec_from_bounds +from common_util.common_util_derived_store import file_stamp from common_util.common_util_json import atomic_write_json from config.config_system import ( SHEET_SURFACE_GRID_M, diff --git a/B06_Section/B06_Section_Router_HaulPlan.py b/B06_Section/B06_Section_Router_HaulPlan.py index b3643bad6..947de6450 100644 --- a/B06_Section/B06_Section_Router_HaulPlan.py +++ b/B06_Section/B06_Section_Router_HaulPlan.py @@ -19,12 +19,14 @@ import hashlib import json import logging from collections import OrderedDict +from pathlib import Path from typing import Any from uuid import UUID from fastapi import APIRouter, Body from fastapi.responses import JSONResponse +from B03_FileInput.B03_FileInput_Repository import get_project_storage_relative_path from B06_Section.B06_Section_Server_Calc_Prebuild import ( BUNDLE, _mass_haul_context, @@ -32,7 +34,10 @@ from B06_Section.B06_Section_Server_Calc_Prebuild import ( haul_inputs_for, haul_limits_for, ) +from common_util.common_util_derived_store import read_derived, write_derived from common_util.common_util_node_bundle import run_bundle_json +from common_util.common_util_storage import resolve_stored_project_path +from config.config_db import run_with_connection logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/projects", tags=["B06 Profile Cross"]) @@ -41,10 +46,11 @@ _NPM_SCRIPT = "build:server-calc" # 측점 수 상한 — 정상 노선은 수백 곳이다. 그보다 크면 비정상 요청으로 본다. _MAX_POINTS = 5000 # 열쇠(누가토량 결과 + 계수 · 장비 경계) → 배분 — 같은 열쇠면 Node 를 다시 안 띄움 -# (342-1 · 부록 4 R25). ponytail: 프로세스 메모리 · 최근 몇 벌만 — 재시작 뒤 첫 열기는 -# 다시 셈(_derived 저장본은 343 틀 뒤). +# (342-1 · 부록 4 R25). 프로세스 메모리 먼저 · 다음 = 프로젝트 `_derived/haul/<열쇠>.json` +# (재시작 뒤 첫 열기도 셈 없음 · 종류당 최근 2벌). _KEPT: OrderedDict[str, Any] = OrderedDict() _KEPT_MAX = 32 +_DERIVED_KIND = "haul" def _plan_key(result: dict[str, Any], context: Any) -> str: @@ -52,6 +58,12 @@ def _plan_key(result: dict[str, Any], context: Any) -> str: return hashlib.sha1(body.encode("utf-8")).hexdigest() +def _keep(key: str, plan: Any) -> None: + _KEPT[key] = plan + if len(_KEPT) > _KEPT_MAX: + _KEPT.popitem(last=False) + + @router.post("/{project_id}/sections/{route_id}/haul-plan", response_model=None) async def compute_haul_plan( project_id: UUID, @@ -87,6 +99,20 @@ async def compute_haul_plan( if key in _KEPT: _KEPT.move_to_end(key) return JSONResponse(content={"status": "success", "haul_plan": _KEPT[key]}) + project_root: Path | None = None + try: + stored_path = await run_with_connection(get_project_storage_relative_path, project_id) + project_root = Path(resolve_stored_project_path(stored_path)) + except Exception: # noqa: BLE001 — 저장소를 못 찾아도 셈은 함(디스크 저장본만 건너뜀) + logger.warning("유토 배분 저장소 못 찾음: project_id=%s", project_id) + stored = ( + None + if project_root is None + else await asyncio.to_thread(read_derived, project_root, _DERIVED_KIND, key) + ) + if stored is not None: + _keep(key, stored["value"]) + return JSONResponse(content={"status": "success", "haul_plan": stored["value"]}) try: output = await asyncio.to_thread( run_bundle_json, @@ -107,7 +133,10 @@ async def compute_haul_plan( content={"status": "error", "message": "유토 배분 계산에 실패했습니다."}, ) plan = output.get("haul_plan") - _KEPT[key] = plan - if len(_KEPT) > _KEPT_MAX: - _KEPT.popitem(last=False) + _keep(key, plan) + try: + if project_root is not None: + await asyncio.to_thread(write_derived, project_root, _DERIVED_KIND, key, plan) + except OSError: # 못 적어도 응답은 그대로 — 다음에 다시 셈 + logger.warning("유토 배분 저장본 쓰기 실패: project_id=%s", project_id, exc_info=True) return JSONResponse(content={"status": "success", "haul_plan": plan}) diff --git a/common_util/common_util_derived_store.py b/common_util/common_util_derived_store.py new file mode 100644 index 000000000..9cc242a97 --- /dev/null +++ b/common_util/common_util_derived_store.py @@ -0,0 +1,78 @@ +"""파생 저장본 — 입력 지문 열쇠 + 결과 한 쌍(343 · 341 부록 4 2-1). + +자리 = 프로젝트 `_derived/<종류>/<열쇠>.json` · 작업본에서 다시 셀 수 있는 값만(정본 아님) · +통째로 지워도 다시 셈으로 돌아옴 · 종류마다 최근 `KEEP` 벌(꺼내면 최근으로). +결과 옆 열쇠(`<이름>.key.json` — 결과가 곧 산출물인 도엽 3D 면 등)는 `key_matches` · `file_stamp`. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from common_util.common_util_json import atomic_write_json + +DERIVED_DIR = "_derived" +KEEP = 2 + + +def derived_key(*parts: Any) -> str: + """입력 지문 — 작은 입력은 내용 그대로 · 파일은 `file_stamp` 를 넣는다.""" + body = json.dumps(parts, sort_keys=True, ensure_ascii=False, default=str) + return hashlib.sha1(body.encode("utf-8")).hexdigest() + + +def file_stamp(path: Path | None) -> str | None: + """파일 지문 = 수정시각 ns : 크기. 없으면 None.""" + try: + stat = path.stat() if path is not None else None + except OSError: + return None + return f"{stat.st_mtime_ns}:{stat.st_size}" if stat else None + + +def key_matches(key_path: Path, key: Any) -> bool: + """결과 옆 열쇠 파일이 지금 입력과 같은지.""" + try: + return json.loads(key_path.read_text(encoding="utf-8")) == key + except (OSError, ValueError): + return False + + +def derived_dir(project_root: Path, kind: str) -> Path: + return Path(project_root) / DERIVED_DIR / kind + + +def read_derived(project_root: Path, kind: str, key: str) -> dict[str, Any] | None: + """저장본 `{"value": …}` — 없거나 못 읽으면 None(값이 null 이어도 맞음과 구별).""" + path = derived_dir(project_root, kind) / f"{key}.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(data, dict) or data.get("key") != key or "value" not in data: + return None + try: + path.touch() # 최근에 쓴 것 = 정리 때 남김 + except OSError: + pass + return data + + +def write_derived(project_root: Path, kind: str, key: str, value: Any, keep: int = KEEP) -> None: + """원자 쓰기 뒤 같은 종류 옛 열쇠를 정리한다.""" + atomic_write_json(derived_dir(project_root, kind) / f"{key}.json", {"key": key, "value": value}) + prune_derived(project_root, kind, keep) + + +def prune_derived(project_root: Path, kind: str, keep: int = KEEP) -> int: + """종류 하나를 최근 `keep` 벌만 남기고 지운다. 지운 수.""" + folder = derived_dir(project_root, kind) + files = sorted( + ((path.stat().st_mtime_ns, path) for path in folder.glob("*.json")), reverse=True + ) + for _, path in files[keep:]: + path.unlink(missing_ok=True) + return max(0, len(files) - keep) diff --git a/resources/tester/test_343_4_derived.py b/resources/tester/test_343_4_derived.py new file mode 100644 index 000000000..a5a799835 --- /dev/null +++ b/resources/tester/test_343_4_derived.py @@ -0,0 +1,170 @@ +"""343 덧 — 파생 저장본 도우미 · 유토 배분 저장본 · B07 읽기 안 지움 · 묶음 낡음 검사 기억.""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +from pathlib import Path +from uuid import uuid4 + +import pytest + +from B06_Section import B06_Section_Router_HaulPlan as haul +from common_util import common_util_derived_store as store +from common_util import common_util_node_bundle as nb + +# ── 도우미 ────────────────────────────────────────────────────────────────────── + + +def test_열쇠_같은_입력_같은_값() -> None: + assert store.derived_key({"a": 1, "b": 2}, 3) == store.derived_key({"b": 2, "a": 1}, 3) + assert store.derived_key({"a": 1}) != store.derived_key({"a": 2}) + + +def test_읽기_쓰기_null_도_맞음(tmp_path: Path) -> None: + assert store.read_derived(tmp_path, "haul", "k") is None + store.write_derived(tmp_path, "haul", "k", None) + assert store.read_derived(tmp_path, "haul", "k") == {"key": "k", "value": None} + assert (tmp_path / "_derived" / "haul" / "k.json").is_file() + + +def test_종류마다_최근_두_벌_꺼내면_최근(tmp_path: Path) -> None: + store.write_derived(tmp_path, "haul", "a", 1) + time.sleep(0.02) + store.write_derived(tmp_path, "haul", "b", 2) + store.write_derived(tmp_path, "other", "x", 0) + time.sleep(0.02) + assert store.read_derived(tmp_path, "haul", "a")["value"] == 1 + time.sleep(0.02) + store.write_derived(tmp_path, "haul", "c", 3) + names = sorted(path.stem for path in (tmp_path / "_derived" / "haul").glob("*.json")) + assert names == ["a", "c"] + assert (tmp_path / "_derived" / "other" / "x.json").is_file() + + +def test_결과_옆_열쇠와_파일_지문(tmp_path: Path) -> None: + path = tmp_path / "m.key.json" + assert not store.key_matches(path, {"a": 1}) + path.write_text(json.dumps({"a": 1}), encoding="utf-8") + assert store.key_matches(path, {"a": 1}) and not store.key_matches(path, {"a": 2}) + assert store.file_stamp(tmp_path / "없음") is None and store.file_stamp(path) + + +# ── 유토 배분 디스크 저장본 ───────────────────────────────────────────────────────── + + +def _wire(monkeypatch: pytest.MonkeyPatch, root: Path, outputs: list) -> list: + calls: list = [] + + async def _none(_project_id): # type: ignore[no-untyped-def] + return {} + + async def _stored(_fn, _project_id): # type: ignore[no-untyped-def] + return "p" + + def _run(_bundle, _script, payload): # type: ignore[no-untyped-def] + calls.append(payload) + return outputs.pop(0) + + monkeypatch.setattr(haul, "haul_inputs_for", _none) + monkeypatch.setattr(haul, "conversion_factors_for", _none) + monkeypatch.setattr(haul, "haul_limits_for", _none) + monkeypatch.setattr(haul, "_mass_haul_context", lambda *_: {"limits": []}) + monkeypatch.setattr(haul, "run_bundle_json", _run) + monkeypatch.setattr(haul, "run_with_connection", _stored) + monkeypatch.setattr(haul, "resolve_stored_project_path", lambda _p: str(root)) + haul._KEPT.clear() + return calls + + +def _post(result: dict) -> dict: + response = asyncio.run(haul.compute_haul_plan(uuid4(), 7, {"result": result})) + return json.loads(response.body) + + +def test_유토_배분_재시작_뒤에도_셈_없음(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + calls = _wire(monkeypatch, tmp_path, [{"haul_plan": {"tag": "a"}}, {"haul_plan": {"tag": "b"}}]) + a = {"points": [{"cumulative_volume_m3": 1.0}]} + assert _post(a)["haul_plan"] == {"tag": "a"} and len(calls) == 1 + key = haul._plan_key(a, {"limits": []}) # 화면 · 서버 메모리와 같은 열쇠 + assert (tmp_path / "_derived" / "haul" / f"{key}.json").is_file() + haul._KEPT.clear() # 재시작 + assert _post(a)["haul_plan"] == {"tag": "a"} and len(calls) == 1 + b = {"points": [{"cumulative_volume_m3": 2.0}]} + assert _post(b)["haul_plan"] == {"tag": "b"} and len(calls) == 2 + + +def test_유토_배분_저장소_못_찾아도_셈(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + calls = _wire(monkeypatch, tmp_path, [{"haul_plan": {"tag": "a"}}]) + + async def _missing(_fn, _project_id): # type: ignore[no-untyped-def] + raise LookupError("없음") + + monkeypatch.setattr(haul, "run_with_connection", _missing) + assert _post({"points": [{"cumulative_volume_m3": 1.0}]})["haul_plan"] == {"tag": "a"} + assert len(calls) == 1 and not (tmp_path / "_derived").exists() + + +# ── B07 읽기는 측점 파일을 안 지움 ───────────────────────────────────────────────── + + +def test_B07_읽기는_잔재를_걸러만_읽음(tmp_path: Path) -> None: + from B07_DesignDetail.B07_DesignDetail_Router_Support_Io import _cross_files + + cross_dir = tmp_path / "B05_Profile" / "cross_sections" + cross_dir.mkdir(parents=True) + for name in ("cross_00000m.json", "cross_00020m.json", "cross_00040m.json"): + (cross_dir / name).write_text("{}", encoding="utf-8") + longitudinal_path = tmp_path / "B05_Profile" / "longitudinal" / "longitudinal.json" + files = _cross_files(longitudinal_path, {"stations": [{"chainage_m": 0}, {"chainage_m": 20}]}) + assert [path.name for path in files] == ["cross_00000m.json", "cross_00020m.json"] + assert (cross_dir / "cross_00040m.json").is_file() + + +# ── 묶음 낡음 검사 기억 ───────────────────────────────────────────────────────── + + +@pytest.fixture +def bundle(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.setattr(nb, "ROOT", tmp_path) + monkeypatch.setattr(nb, "_FRESH", {}) + path = tmp_path / "dist" / "bundle.mjs" + path.parent.mkdir() + path.write_text("", encoding="utf-8") + os.utime(path, (1_000_000_000, 1_000_000_000)) + return path + + +def _counting(monkeypatch: pytest.MonkeyPatch) -> list: + scans: list = [] + real = nb._sources_newer + + def counting(built_at: float) -> bool: + scans.append(built_at) + return real(built_at) + + monkeypatch.setattr(nb, "_sources_newer", counting) + return scans + + +def test_낡지_않음은_잠시_기억(bundle: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scans = _counting(monkeypatch) + assert not nb.bundle_stale(bundle) and not nb.bundle_stale(bundle) + assert len(scans) == 1 + # 묶음을 새로 지으면(mtime 바뀜) 다시 잼 + os.utime(bundle, (1_000_000_100, 1_000_000_100)) + assert not nb.bundle_stale(bundle) and len(scans) == 2 + # 기억 시간이 지나면 다시 잼 + monkeypatch.setattr(nb, "RECHECK_S", 0.0) + assert not nb.bundle_stale(bundle) and len(scans) == 3 + + +def test_낡음은_기억하지_않음(bundle: Path, monkeypatch: pytest.MonkeyPatch) -> None: + source = nb.ROOT / "B05_Profile" / "x.ts" + source.parent.mkdir() + source.write_text("", encoding="utf-8") + scans = _counting(monkeypatch) + assert nb.bundle_stale(bundle) and nb.bundle_stale(bundle) + assert len(scans) == 2 and str(bundle) not in nb._FRESH