58 lines
2.2 KiB
Python
58 lines
2.2 KiB
Python
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
# Load master
|
|
master_path = Path('resources/data_work_item_master/work_item_master_2026-01-01.json')
|
|
with open(master_path, 'r', encoding='utf-8') as f:
|
|
master = json.load(f)
|
|
|
|
# Load pum_forest
|
|
pum_path = Path('resources/data_cost_input_value/pum_forest_2026.json')
|
|
with open(pum_path, 'r', encoding='utf-8') as f:
|
|
pum_data = json.load(f)
|
|
|
|
pum_tables = pum_data.get('variables', {}).get('pum', {}).get('tables', [])
|
|
|
|
print(f"pum_tables count: {len(pum_tables)}")
|
|
print(f"master work_items: {len(master['work_items'])}")
|
|
master_tables = []
|
|
for wi in master['work_items']:
|
|
master_tables.extend(wi.get('tables', []))
|
|
orphan_tables = master.get('orphan_tables', [])
|
|
print(f"master tables in work_items: {len(master_tables)}")
|
|
print(f"master orphan_tables: {len(orphan_tables)}")
|
|
total_master_tables = master_tables + orphan_tables
|
|
print(f"total master tables: {len(total_master_tables)}")
|
|
|
|
# Compare pum_tables and master_tables by pum_table_id
|
|
pum_ids = {t['pum_table_id'] for t in pum_tables if 'pum_table_id' in t}
|
|
master_ids = {t['pum_table_id'] for t in total_master_tables if 'pum_table_id' in t}
|
|
|
|
print(f"Unique table IDs in pum_forest: {len(pum_ids)}")
|
|
print(f"Unique table IDs in master: {len(master_ids)}")
|
|
diff_pum_master = pum_ids - master_ids
|
|
diff_master_pum = master_ids - pum_ids
|
|
print(f"IDs in pum_forest but NOT in master: {diff_pum_master}")
|
|
print(f"IDs in master but NOT in pum_forest: {diff_master_pum}")
|
|
|
|
# Check sample pum_table
|
|
t0 = pum_tables[0]
|
|
print("\nSample pum_table keys:", list(t0.keys()))
|
|
print("Sample pum_table metadata:")
|
|
for k in ['pum_table_id', 'section', 'source_line', 'pum_form', 'form_basis', 'basis_quantity', 'basis_unit', 'basis_source']:
|
|
print(f" {k}: {t0.get(k)}")
|
|
|
|
# Check raw_markdown presence
|
|
has_raw = sum(1 for t in pum_tables if 'raw_markdown' in t and t['raw_markdown'])
|
|
print(f"\nTables with raw_markdown in pum_forest: {has_raw}/{len(pum_tables)}")
|
|
|
|
# Check pum_form distribution
|
|
from collections import Counter
|
|
form_counts = Counter(t.get('pum_form') for t in pum_tables)
|
|
print("\npum_form distribution in pum_forest:")
|
|
for form, cnt in form_counts.most_common():
|
|
print(f" {form}: {cnt}")
|