71 lines
2.8 KiB
Python
71 lines
2.8 KiB
Python
import os, sys, re, json
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
base_dir = "resources/knowledge/original/원가계산/건설공사_표준품셈"
|
|
|
|
# Exclude 8장 (건설기계) and 합본
|
|
all_md_files = []
|
|
for root, dirs, files in os.walk(base_dir):
|
|
for f in files:
|
|
if f.endswith('.md') and not f.startswith('_') and '개정사항' not in f and '2026년_건설공사_표준품셈.md' not in f:
|
|
if '01_공통부문\\제8장' in os.path.join(root, f) or '01_공통부문/제8장' in os.path.join(root, f):
|
|
continue
|
|
all_md_files.append(os.path.join(root, f))
|
|
|
|
print(f"Total target files (excluding 8장 and merged): {len(all_md_files)}")
|
|
|
|
results = []
|
|
|
|
for fpath in sorted(all_md_files):
|
|
rel_path = os.path.relpath(fpath, base_dir)
|
|
with open(fpath, 'r', encoding='utf-8') as f:
|
|
lines = f.readlines()
|
|
|
|
for idx, line in enumerate(lines):
|
|
line_num = idx + 1
|
|
line_str = line.strip()
|
|
if not (line_str.startswith('|') and line_str.endswith('|')):
|
|
continue
|
|
|
|
cells = [c.strip() for c in line_str.split('|')[1:-1]]
|
|
# Skip header separators |---|---|
|
|
if all(re.match(r'^:?-+:?$', c) for c in cells if c):
|
|
continue
|
|
|
|
# Check for multi-token collapsed cells
|
|
# Conditions:
|
|
# A) cell has >= 8 tokens and contains numbers or spec symbols
|
|
# B) cell contains embedded table headers like '구 분', '규 격', '단 위' etc
|
|
# C) multiple numeric tokens that look like a row data squashed together
|
|
for c_idx, c in enumerate(cells):
|
|
tokens = c.split()
|
|
# If cell has embedded table structure
|
|
is_embedded_table = ('구 분' in c or '구분' in c) and ('단위' in c or '단 위' in c or '수량' in c or '수 량' in c)
|
|
# If cell has many tokens
|
|
is_many_tokens = len(tokens) >= 8 and sum(1 for tok in tokens if any(char.isdigit() for char in tok)) >= 4
|
|
|
|
if is_embedded_table or is_many_tokens:
|
|
results.append({
|
|
"file": rel_path,
|
|
"line": line_num,
|
|
"col_idx": c_idx,
|
|
"cell_snippet": c[:120],
|
|
"token_count": len(tokens),
|
|
"is_embedded_table": is_embedded_table
|
|
})
|
|
break
|
|
|
|
print(f"Total candidate collapsed rows found: {len(results)}")
|
|
|
|
by_file = {}
|
|
for r in results:
|
|
f = r['file']
|
|
by_file[f] = by_file.get(f, 0) + 1
|
|
|
|
for f, cnt in sorted(by_file.items(), key=lambda x: x[1], reverse=True):
|
|
print(f" {cnt:3d} rows: {f}")
|
|
|
|
with open('scratch/all_candidates_collapsed_ex8.json', 'w', encoding='utf-8') as f:
|
|
json.dump(results, f, ensure_ascii=False, indent=2)
|