122 lines
6.0 KiB
Python
122 lines
6.0 KiB
Python
import os, sys, re
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
target_files = [
|
|
# 02_토목부문
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제1장_도로포장공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제2장_하천공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제3장_터널공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제4장_궤도공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제5장_강구조공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제6장_관부설및접합공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제7장_항만공사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제8장_지반조사.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/02_토목부문/제9장_측량.md',
|
|
# 05_유지관리부문
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제1장_공통.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제2장_토목.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제3장_건축.md',
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/05_유지관리부문/제4장_기계설비.md',
|
|
# 04_기계설비부문 제13장
|
|
'resources/knowledge/original/원가계산/건설공사_표준품셈/04_기계설비부문/제13장_플랜트설비공사.md',
|
|
]
|
|
|
|
print("=== Scanning by Section for Overlapping / Redundant Table Rows ===")
|
|
|
|
findings = []
|
|
|
|
for fpath in target_files:
|
|
fname = os.path.basename(fpath)
|
|
with open(fpath, 'r', encoding='utf-8') as f:
|
|
content = f.read()
|
|
|
|
# Split content by section pattern: e.g. \n\d+-\d+(?:-\d+)?
|
|
# Or line by line tracking current section
|
|
lines = content.split('\n')
|
|
current_sec = "TOP"
|
|
sec_lines = []
|
|
|
|
sections = [] # (sec_name, start_lno, lines)
|
|
for idx, line in enumerate(lines):
|
|
# Section header match e.g. "1-2-4 ", "## 1-2-4", etc.
|
|
m = re.match(r'^(?:#+\s*)?(\d+-\d+(?:-\d+)?)\s+', line.strip())
|
|
if m:
|
|
if sec_lines:
|
|
sections.append((current_sec, idx - len(sec_lines) + 1, sec_lines))
|
|
current_sec = m.group(1)
|
|
sec_lines = [line]
|
|
else:
|
|
sec_lines.append(line)
|
|
if sec_lines:
|
|
sections.append((current_sec, len(lines) - len(sec_lines) + 1, sec_lines))
|
|
|
|
for sec_name, start_lno, s_lines in sections:
|
|
# Extract tables in this section
|
|
tables = []
|
|
cur_t = []
|
|
cur_t_start = 0
|
|
for s_idx, sl in enumerate(s_lines):
|
|
l_str = sl.strip()
|
|
if l_str.startswith('|') and l_str.endswith('|'):
|
|
if not cur_t:
|
|
cur_t_start = start_lno + s_idx
|
|
cur_t.append((start_lno + s_idx, l_str))
|
|
else:
|
|
if cur_t:
|
|
tables.append((cur_t_start, cur_t))
|
|
cur_t = []
|
|
if cur_t:
|
|
tables.append((cur_t_start, cur_t))
|
|
|
|
# Check within each table or across tables in the same section
|
|
# Collect all rows in this section
|
|
all_rows = []
|
|
for t_start, t_rows in tables:
|
|
for r_lno, r_str in t_rows:
|
|
cells = [c.strip() for c in r_str.split('|')[1:-1]]
|
|
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
|
continue
|
|
if cells and cells[0] in ["비고", "비 고", "주", "[주]"]:
|
|
continue
|
|
# Skip header rows
|
|
if any(c in ["구 분", "구분", "품 명", "품명", "규 격", "규격", "단 위", "단위", "수 량", "수량", "직 종", "직종"] for c in cells):
|
|
continue
|
|
all_rows.append((r_lno, r_str, cells))
|
|
|
|
# Now check if later rows duplicate earlier rows in the same section!
|
|
# Two types:
|
|
# Type A: Exact same data in 3 or more consecutive rows
|
|
# Type B: A single row that squashes 2 or more preceding rows from the same section
|
|
for i in range(len(all_rows)):
|
|
r_lno, r_str, cells = all_rows[i]
|
|
|
|
# Check if this row is a squashed version of preceding rows
|
|
# Extract distinct tokens from this row
|
|
row_tokens = set(re.findall(r'[가-힣A-Za-z0-9.]+', r_str))
|
|
row_tokens = {t for t in row_tokens if len(t) > 1 and t not in ["인부", "보통인부", "특별인부", "0.0", "1.0"]}
|
|
|
|
matching_preceding = []
|
|
for j in range(0, i):
|
|
prev_lno, prev_str, prev_cells = all_rows[j]
|
|
prev_tokens = set(re.findall(r'[가-힣A-Za-z0-9.]+', prev_str))
|
|
prev_tokens = {t for t in prev_tokens if len(t) > 1 and t not in ["인부", "보통인부", "특별인부", "0.0", "1.0"]}
|
|
|
|
# Check how much prev_tokens is contained in row_tokens
|
|
if len(prev_tokens) >= 3 and prev_tokens.issubset(row_tokens):
|
|
matching_preceding.append(prev_lno)
|
|
|
|
if len(matching_preceding) >= 2:
|
|
findings.append({
|
|
'file': fname,
|
|
'section': sec_name,
|
|
'line': r_lno,
|
|
'squashed_row': r_str[:80],
|
|
'duplicates_of_lines': matching_preceding
|
|
})
|
|
|
|
print(f"Total findings of squashed rows duplicating preceding rows in same section: {len(findings)}")
|
|
for f in findings:
|
|
print(f"[{f['file']}:{f['section']}:L{f['line']}] duplicates lines {f['duplicates_of_lines']}")
|
|
print(f" Squashed row: {f['squashed_row']}")
|