import json
import sys
from pathlib import Path

sys.stdout.reconfigure(encoding='utf-8')

# Load master
master_path = Path('resources/data_work_item_master/work_item_master_2026-01-01.json')
with open(master_path, 'r', encoding='utf-8') as f:
    master = json.load(f)

# Load pum_forest
pum_path = Path('resources/data_cost_input_value/pum_forest_2026.json')
with open(pum_path, 'r', encoding='utf-8') as f:
    pum_data = json.load(f)

pum_tables = pum_data.get('variables', {}).get('pum', {}).get('tables', [])

print(f"pum_tables count: {len(pum_tables)}")
print(f"master work_items: {len(master['work_items'])}")
master_tables = []
for wi in master['work_items']:
    master_tables.extend(wi.get('tables', []))
orphan_tables = master.get('orphan_tables', [])
print(f"master tables in work_items: {len(master_tables)}")
print(f"master orphan_tables: {len(orphan_tables)}")
total_master_tables = master_tables + orphan_tables
print(f"total master tables: {len(total_master_tables)}")

# Compare pum_tables and master_tables by pum_table_id
pum_ids = {t['pum_table_id'] for t in pum_tables if 'pum_table_id' in t}
master_ids = {t['pum_table_id'] for t in total_master_tables if 'pum_table_id' in t}

print(f"Unique table IDs in pum_forest: {len(pum_ids)}")
print(f"Unique table IDs in master: {len(master_ids)}")
diff_pum_master = pum_ids - master_ids
diff_master_pum = master_ids - pum_ids
print(f"IDs in pum_forest but NOT in master: {diff_pum_master}")
print(f"IDs in master but NOT in pum_forest: {diff_master_pum}")

# Check sample pum_table
t0 = pum_tables[0]
print("\nSample pum_table keys:", list(t0.keys()))
print("Sample pum_table metadata:")
for k in ['pum_table_id', 'section', 'source_line', 'pum_form', 'form_basis', 'basis_quantity', 'basis_unit', 'basis_source']:
    print(f"  {k}: {t0.get(k)}")

# Check raw_markdown presence
has_raw = sum(1 for t in pum_tables if 'raw_markdown' in t and t['raw_markdown'])
print(f"\nTables with raw_markdown in pum_forest: {has_raw}/{len(pum_tables)}")

# Check pum_form distribution
from collections import Counter
form_counts = Counter(t.get('pum_form') for t in pum_tables)
print("\npum_form distribution in pum_forest:")
for form, cnt in form_counts.most_common():
    print(f"  {form}: {cnt}")