127 lines
6.0 KiB
Python
127 lines
6.0 KiB
Python
import os, sys, re
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
target_files = [
|
|
# 02_토목부문
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
|
|
# 05_유지관리부문
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
|
|
# 04_기계설비부문 제13장
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
|
|
]
|
|
|
|
print("=== Scanning 14 target files for duplicated/overlapped rows ===")
|
|
|
|
exact_duplicates = []
|
|
squashed_overlaps = []
|
|
header_repeats = []
|
|
|
|
for fpath in target_files:
|
|
fname = os.path.basename(fpath)
|
|
with open(fpath, 'r', encoding='utf-8') as f:
|
|
lines = f.readlines()
|
|
|
|
# Track recent data rows (sliding window of 40 rows)
|
|
recent_rows = [] # list of (line_no, line_str, cells, numbers_set, text_words_set)
|
|
recent_headers = []
|
|
|
|
in_table = False
|
|
|
|
for idx, line in enumerate(lines):
|
|
line_str = line.strip()
|
|
l_no = idx + 1
|
|
|
|
if not (line_str.startswith('|') and line_str.endswith('|')):
|
|
in_table = False
|
|
# If leaving table, keep recent_rows for a bit or clear?
|
|
# A duplicate table might start right after!
|
|
continue
|
|
|
|
cells = [c.strip() for c in line_str.split('|')[1:-1]]
|
|
is_divider = all(re.match(r'^:?-+:?$', c) for c in cells if c)
|
|
if is_divider:
|
|
continue
|
|
|
|
# Is it a header row?
|
|
is_header = any(c in ["구 분", "구분", "품 명", "품명", "규 격", "규격", "단 위", "단위", "수 량", "수량", "직 종", "직종"] for c in cells)
|
|
if is_header:
|
|
# Check if identical header was seen recently (within 50 lines)
|
|
for h_lno, h_cells in recent_headers[-5:]:
|
|
if h_cells == cells:
|
|
header_repeats.append({
|
|
'file': fname,
|
|
'line': l_no,
|
|
'prev_line': h_lno,
|
|
'header': cells
|
|
})
|
|
recent_headers.append((l_no, cells))
|
|
continue
|
|
|
|
# Skip notes
|
|
if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
|
|
continue
|
|
|
|
# Extract numbers and words
|
|
row_numbers = set(re.findall(r'\b\d+(?:\.\d+)?\b', line_str))
|
|
row_words = set(w for w in re.findall(r'[가-힣A-Za-z0-9]+', line_str) if len(w) > 1)
|
|
|
|
# 1. Check exact duplicate in sliding window
|
|
for r_lno, r_str, r_cells, r_nums, r_words in recent_rows[-30:]:
|
|
# If cells match exactly, and not just all empty or standard dashes
|
|
non_empty = [c for c in cells if c not in ["", "-", "〃"]]
|
|
if len(non_empty) >= 2 and cells == r_cells:
|
|
exact_duplicates.append({
|
|
'file': fname,
|
|
'line': l_no,
|
|
'prev_line': r_lno,
|
|
'row': line_str[:80]
|
|
})
|
|
break
|
|
|
|
# 2. Check squashed overlap:
|
|
# If this row contains multiple words/numbers that appeared across MULTIPLE preceding separate rows
|
|
# E.g., row_words has words from at least 3 distinct preceding rows, or numbers from 3 distinct rows
|
|
matches_per_row = []
|
|
for r_lno, r_str, r_cells, r_nums, r_words in recent_rows[-20:]:
|
|
common_nums = row_numbers.intersection(r_nums)
|
|
common_words = row_words.intersection(r_words)
|
|
if len(common_nums) >= 2 or len(common_words) >= 2:
|
|
matches_per_row.append((r_lno, common_words, common_nums))
|
|
|
|
if len(matches_per_row) >= 3:
|
|
# This row overlaps with 3 or more distinct preceding rows!
|
|
squashed_overlaps.append({
|
|
'file': fname,
|
|
'line': l_no,
|
|
'matched_prev_lines': [m[0] for m in matches_per_row],
|
|
'row': line_str[:90]
|
|
})
|
|
|
|
recent_rows.append((l_no, line_str, cells, row_numbers, row_words))
|
|
if len(recent_rows) > 40:
|
|
recent_rows.pop(0)
|
|
|
|
print(f"\n1. Exact duplicate rows in window: {len(exact_duplicates)}")
|
|
for d in exact_duplicates:
|
|
print(f" [{d['file']}:L{d['line']} (prev L{d['prev_line']})] {d['row']}")
|
|
|
|
print(f"\n2. Squashed overlaps (matching 3+ preceding rows): {len(squashed_overlaps)}")
|
|
for s in squashed_overlaps:
|
|
print(f" [{s['file']}:L{s['line']} (matched {s['matched_prev_lines']})] {s['row']}")
|
|
|
|
print(f"\n3. Header repeats (same table header repeated within 50 lines): {len(header_repeats)}")
|
|
for h in header_repeats:
|
|
print(f" [{h['file']}:L{h['line']} (prev L{h['prev_line']})] {h['header']}")
|