113 lines
5.5 KiB
Python
113 lines
5.5 KiB
Python
import os, sys, re
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
target_files = [
|
|
# 02_토목부문 (전체 9개 파일)
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
|
|
# 05_유지관리부문 (전체 4개 파일)
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
|
|
# 04_기계설비부문 제13장
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
|
|
]
|
|
|
|
print("=== Scanning Tables for Trailing Duplicate/Overlapped Rows ===")
|
|
|
|
tail_duplicates = []
|
|
|
|
for fpath in target_files:
|
|
fname = os.path.basename(fpath)
|
|
with open(fpath, 'r', encoding='utf-8') as f:
|
|
lines = f.readlines()
|
|
|
|
tables = []
|
|
cur_t = []
|
|
cur_start = 0
|
|
for idx, l in enumerate(lines):
|
|
line_str = l.strip()
|
|
if line_str.startswith('|') and line_str.endswith('|'):
|
|
if not cur_t:
|
|
cur_start = idx + 1
|
|
cur_t.append((idx + 1, line_str))
|
|
else:
|
|
if cur_t:
|
|
tables.append((cur_start, cur_t))
|
|
cur_t = []
|
|
if cur_t:
|
|
tables.append((cur_start, cur_t))
|
|
|
|
for t_start, t_rows in tables:
|
|
# Separate divider
|
|
data_rows = []
|
|
for lno, r_str in t_rows:
|
|
cells = [c.strip() for c in r_str.split('|')[1:-1]]
|
|
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
|
continue
|
|
if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
|
|
continue
|
|
data_rows.append((lno, r_str, cells))
|
|
|
|
if len(data_rows) < 4:
|
|
continue
|
|
|
|
# Check every row against all PRECEDING rows in the table
|
|
# We look for:
|
|
# 1. Exact duplicate rows (not consecutive 〃 or standard repeats)
|
|
# 2. Rows whose cells contain multiple tokens from distinct preceding rows
|
|
|
|
seen_cells_set = [] # list of (lno, cells, text_tokens, num_tokens)
|
|
|
|
for r_idx, (lno, r_str, cells) in enumerate(data_rows):
|
|
# Tokens
|
|
text_tokens = set(re.findall(r'[가-힣A-Za-z]+', r_str))
|
|
text_tokens = {t for t in text_tokens if len(t) > 1 and t not in ["인부", "보통인부", "특별인부", "플랜트", "설비공", "용접공", "배관공"]}
|
|
num_tokens = set(re.findall(r'\b\d+(?:\.\d+)?\b', r_str))
|
|
|
|
# Check 1: Exact duplicate of an earlier row that is NOT adjacent
|
|
for prev_lno, prev_cells, prev_tt, prev_nt in seen_cells_set[:-1]:
|
|
non_empty = [c for c in cells if c not in ["", "-", "〃"]]
|
|
if len(non_empty) >= 2 and cells == prev_cells:
|
|
tail_duplicates.append({
|
|
'file': fname,
|
|
'line': lno,
|
|
'type': 'exact_duplicate_of_earlier_row',
|
|
'dup_of_line': prev_lno,
|
|
'row': r_str[:80]
|
|
})
|
|
break
|
|
|
|
# Check 2: Squashed composite row that combines tokens of 2+ distinct preceding rows
|
|
matching_earlier = []
|
|
for prev_lno, prev_cells, prev_tt, prev_nt in seen_cells_set:
|
|
if len(prev_tt) >= 2 and prev_tt.issubset(text_tokens) and len(prev_nt.intersection(num_tokens)) >= 1:
|
|
matching_earlier.append(prev_lno)
|
|
|
|
if len(matching_earlier) >= 2:
|
|
tail_duplicates.append({
|
|
'file': fname,
|
|
'line': lno,
|
|
'type': 'composite_squashed_duplicate',
|
|
'dup_of_lines': matching_earlier,
|
|
'row': r_str[:80]
|
|
})
|
|
|
|
seen_cells_set.append((lno, cells, text_tokens, num_tokens))
|
|
|
|
print(f"Total potential trailing duplicates found: {len(tail_duplicates)}")
|
|
for td in tail_duplicates:
|
|
if td['type'] == 'exact_duplicate_of_earlier_row':
|
|
print(f"[{td['file']}:L{td['line']}] EXACT DUP of L{td['dup_of_line']}: {td['row']}")
|
|
else:
|
|
print(f"[{td['file']}:L{td['line']}] COMPOSITE DUP of {td['dup_of_lines']}: {td['row']}")
|