perf(863-2): 구조화 저장본 · 배수 격자 배열 압축 1 단

- 구조화 저장본(structured.npz) · 배수 격자 배열 저장을 savez_compressed(압축 6 단) → 같은 zip 꼴 압축 1 단(save_npz_fast)
- 실측(검증_sub2 점군 4,900 만 점) — 쓰기 99.2 → 38.4 초 · 크기 841 → 865 MB · 다시 읽은 값 열쇠마다 같음 · 읽기 9.6 → 10.0 초
- 무압축(1.6 초 · 1,568 MB)은 저장소 동기화 크기 1.9 배라 안 고름
- 읽는 쪽 그대로(np.load) · 옛 저장본도 그대로 읽힘 · 시험 test_863_2_npz_fast

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-21 14:12:54 +09:00
co-authored by Claude Opus 5.5
parent 277ecac99e
commit 8d79797509
4 changed files with 63 additions and 3 deletions
@@ -19,6 +19,7 @@ from typing import Any
import laspy
import numpy as np
from common_util.common_util_atomic import save_npz_fast
from common_util.common_util_json import replace_with_retry
from config.config_system import (
SURFACE_DEFAULT_RGB_VALUE,
@@ -314,7 +315,7 @@ def structurize_las(
delete=False,
) as temporary:
temporary_path = Path(temporary.name)
np.savez_compressed(
save_npz_fast(
temporary,
**merged.arrays(),
bounds=bounds,
@@ -18,9 +18,9 @@ from collections.abc import Sequence
from pathlib import Path
from typing import Any, Callable
import numpy as np
from shapely.geometry.base import BaseGeometry
from common_util.common_util_atomic import save_npz_fast
from common_util.common_util_storage import resolve_stored_project_path
from common_util.common_util_time import utc_now_text
from config.config_system import DRAINAGE_CACHE_DIRNAME
@@ -148,7 +148,7 @@ def write_grid_arrays(
target = directory / f"{prefix}_{stage}.npz"
try:
directory.mkdir(parents=True, exist_ok=True)
np.savez_compressed(
save_npz_fast(
target,
x_min=spec.x_min,
y_max=spec.y_max,
+15
View File
@@ -2,11 +2,26 @@
import os
import tempfile
import zipfile
from pathlib import Path
from typing import IO, Any
import numpy as np
def save_npz_fast(file: str | Path | IO[bytes], **arrays: Any) -> None:
"""`np.savez_compressed` 와 같은 꼴(deflate zip) — 압축 1 단이라 큰 점군에서 약 2.6 배 빠름.
크기는 거의 같음(841 → 865 MB) · 읽기는 `np.load` 그대로.
"""
with zipfile.ZipFile(
file, "w", zipfile.ZIP_DEFLATED, allowZip64=True, compresslevel=1
) as archive:
for name, value in arrays.items():
with archive.open(f"{name}.npy", "w", force_zip64=True) as member:
np.lib.format.write_array(member, np.asanyarray(value))
def atomic_write_bytes(path: str | Path, payload: bytes) -> None:
"""같은 디렉터리의 임시 파일을 교체하여 바이트를 원자적으로 저장한다."""
target = Path(path)
+44
View File
@@ -0,0 +1,44 @@
"""863-2 압축 1 단 npz — np.load 로 같은 값 · 옛 savez_compressed 와 같은 열쇠 · 파일 객체에도 씀."""
from __future__ import annotations
import io
import zipfile
from pathlib import Path
import numpy as np
from common_util.common_util_atomic import save_npz_fast
def _arrays() -> dict[str, np.ndarray]:
rng = np.random.default_rng(863)
return {
"xyz": rng.uniform(0, 1000, (5000, 3)),
"classification": rng.integers(0, 8, 5000).astype(np.uint8),
"bounds": np.zeros((3, 2)),
"cell_m": np.float64(2.0),
"n_rows": 12,
}
def test_다시_읽으면_같은_값(tmp_path: Path) -> None:
arrays = _arrays()
fast, old = tmp_path / "fast.npz", tmp_path / "old.npz"
save_npz_fast(fast, **arrays)
np.savez_compressed(old, **arrays)
with np.load(fast) as a, np.load(old) as b:
assert sorted(a.files) == sorted(b.files)
for key in a.files:
assert a[key].dtype == b[key].dtype
assert np.array_equal(a[key], b[key])
with zipfile.ZipFile(fast) as archive:
assert all(i.compress_type == zipfile.ZIP_DEFLATED for i in archive.infolist())
def test_파일_객체에도_씀() -> None:
buffer = io.BytesIO()
save_npz_fast(buffer, xyz=_arrays()["xyz"])
buffer.seek(0)
with np.load(buffer) as loaded:
assert np.array_equal(loaded["xyz"], _arrays()["xyz"])