58 lines
2.5 KiB
Python
58 lines
2.5 KiB
Python
import os, sys, re, json
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
base_dir = "resources/knowledge/original/원가계산/건설공사_표준품셈"
|
|
|
|
all_md_files = []
|
|
for root, dirs, files in os.walk(base_dir):
|
|
for f in files:
|
|
if f.endswith('.md') and not f.startswith('_') and '개정사항' not in f and '2026년_건설공사_표준품셈.md' not in f:
|
|
if '01_공통부문\\제8장' in os.path.join(root, f) or '01_공통부문/제8장' in os.path.join(root, f):
|
|
continue
|
|
all_md_files.append(os.path.join(root, f))
|
|
|
|
squashed_tables = []
|
|
|
|
for fpath in sorted(all_md_files):
|
|
rel_path = os.path.relpath(fpath, base_dir)
|
|
with open(fpath, 'r', encoding='utf-8') as f:
|
|
lines = f.readlines()
|
|
|
|
for idx, line in enumerate(lines):
|
|
line_num = idx + 1
|
|
line_str = line.strip()
|
|
if line_str.startswith('|'):
|
|
continue
|
|
|
|
# Check if line contains a squashed table
|
|
# Sign 1: contains both '구 분' (or '구분') and ('단위' or '단 위') and ('수량' or '수 량')
|
|
# Sign 2: contains sec header followed immediately by table elements
|
|
has_table_headers = ('구 분' in line_str or '구분' in line_str) and \
|
|
('단 위' in line_str or '단위' in line_str) and \
|
|
('수 량' in line_str or '수량' in line_str or '인' in line_str)
|
|
|
|
# Also check for tables without '수량' like '규격' '단위'
|
|
has_spec_unit = ('규 격' in line_str or '규격' in line_str) and \
|
|
('단 위' in line_str or '단위' in line_str) and \
|
|
any(term in line_str for term in ['공(인)', '인부', '비고', '시공량'])
|
|
|
|
if has_table_headers or has_spec_unit:
|
|
# Check length to avoid simple sentences
|
|
if len(line_str) > 40:
|
|
squashed_tables.append({
|
|
"file": rel_path,
|
|
"line": line_num,
|
|
"length": len(line_str),
|
|
"line_text": line_str
|
|
})
|
|
|
|
print(f"Total squashed table lines in non-table text: {len(squashed_tables)}")
|
|
|
|
for st in squashed_tables:
|
|
print(f"\n{st['file']} L{st['line']} (len: {st['length']}):")
|
|
print(f" {st['line_text'][:120]}")
|
|
|
|
with open('scratch/squashed_tables_in_text.json', 'w', encoding='utf-8') as f:
|
|
json.dump(squashed_tables, f, ensure_ascii=False, indent=2)
|