104 lines
5.4 KiB
Python
104 lines
5.4 KiB
Python
import os, sys, re
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
target_files = [
|
|
# 02_토목부문
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
|
|
# 05_유지관리부문
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
|
|
# 04_기계설비부문 제13장
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
|
|
]
|
|
|
|
print("=== Strict Audit of Duplicate Substantive Data Rows across 14 Target Chapters ===")
|
|
|
|
strict_candidates = []
|
|
|
|
for fpath in target_files:
|
|
fname = os.path.basename(fpath)
|
|
with open(fpath, 'r', encoding='utf-8') as f:
|
|
lines = f.readlines()
|
|
|
|
# Collect all table rows
|
|
all_rows = []
|
|
for idx, l in enumerate(lines):
|
|
line_str = l.strip()
|
|
if not (line_str.startswith('|') and line_str.endswith('|')):
|
|
continue
|
|
cells = [c.strip() for c in line_str.split('|')[1:-1]]
|
|
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
|
continue
|
|
if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
|
|
continue
|
|
if any(c in ["구 분", "구분", "품 명", "품명", "규 격", "규격", "단 위", "단위", "수 량", "수량", "직 종", "직종"] for c in cells):
|
|
continue
|
|
|
|
# Is it a substantive row?
|
|
# Must have at least one specific item name, spec, or multi-values
|
|
# Filter out generic worker rows with no spec (e.g. '| 보통인부 | 인 | 1 | |')
|
|
non_empty = [c for c in cells if c and c not in ["-", "〃"]]
|
|
if len(non_empty) <= 1:
|
|
continue
|
|
|
|
# Check if it's just a generic worker row
|
|
if len(non_empty) <= 3 and any(w in line_str for w in ["보통인부", "특별인부", "인부", "기계설비공", "용접공"]) and not any(re.search(r'\d+\.\d{2,}', c) for c in cells):
|
|
# Check if all numbers are small ints like 1, 2, 3
|
|
nums = re.findall(r'\b\d+(?:\.\d+)?\b', line_str)
|
|
if all(n in ["1", "2", "3", "4", "5", "10", "20"] for n in nums):
|
|
continue
|
|
|
|
all_rows.append((idx + 1, line_str, cells))
|
|
|
|
# Now check for duplicates within distance of 80 rows
|
|
for i in range(len(all_rows)):
|
|
lno_i, str_i, cells_i = all_rows[i]
|
|
for j in range(i + 1, min(len(all_rows), i + 80)):
|
|
lno_j, str_j, cells_j = all_rows[j]
|
|
|
|
# Exact match of non-trivial row
|
|
if cells_i == cells_j:
|
|
strict_candidates.append({
|
|
'file': fname,
|
|
'line_1': lno_i,
|
|
'line_2': lno_j,
|
|
'type': 'exact_duplicate_substantive',
|
|
'row': str_i[:90]
|
|
})
|
|
# Check if str_j is a squashed row containing str_i
|
|
# E.g. cells_j has multiple values per cell and covers cells_i
|
|
else:
|
|
words_i = set(re.findall(r'[가-힣A-Za-z0-9.]+', str_i))
|
|
words_j = set(re.findall(r'[가-힣A-Za-z0-9.]+', str_j))
|
|
# Remove generic tokens
|
|
clean_i = {w for w in words_i if w not in ["인부", "보통인부", "특별인부", "인", "대", "개", "m", "㎡", "㎥", "ton"]}
|
|
if len(clean_i) >= 4 and clean_i.issubset(words_j) and len(words_j) > len(words_i) + 4:
|
|
strict_candidates.append({
|
|
'file': fname,
|
|
'line_1': lno_i,
|
|
'line_2': lno_j,
|
|
'type': 'squashed_superset_of_line_1',
|
|
'row_1': str_i[:80],
|
|
'row_2': str_j[:80]
|
|
})
|
|
|
|
print(f"Total strict candidates found: {len(strict_candidates)}")
|
|
for sc in strict_candidates:
|
|
print(f"\n[{sc['file']}] L{sc['line_1']} & L{sc['line_2']} - {sc['type']}")
|
|
if sc['type'] == 'exact_duplicate_substantive':
|
|
print(f" Row: {sc['row']}")
|
|
else:
|
|
print(f" Row 1: {sc['row_1']}")
|
|
print(f" Row 2: {sc['row_2']}")
|