#!/usr/bin/env python3
"""
公司认定方案清洗 v2 — 完整提取 3工作簿 × 2类工作表
  表1-专项施工方案 (危大) → certified_schemes_detail.csv
  表2-I II Ⅲ类技术方案    → certified_tech_schemes_detail.csv
输入: 附件1：技术方案（含专项施工方案）编制计划表 (3个xlsx)
输出: dashboard/data/认定数据/2026/ (年度固定目录)
"""
import pandas as pd
import openpyxl
from pathlib import Path
import json
import difflib
from datetime import datetime

# ═══ CONFIG ═══
RAW_DIR = Path("/mnt/y/WorkingEmail/OA收文_Incoming")
CERT_SRC_DIR = RAW_DIR / "2026-03-16_关于公布公司2026年度技术方案编制计划的通知"
OUT_DIR = Path("/mnt/y/Openclaw_Hub/03.资源/实施项目 wiki/dashboard/data/认定数据/2026")
OUT_DIR.mkdir(parents=True, exist_ok=True)

CERT_FILES = [
    CERT_SRC_DIR / "附件1：技术方案（含专项施工方案）编制计划表-建筑类项目.xlsx",
    CERT_SRC_DIR / "附件1：技术方案（含专项施工方案）编制计划表-非建筑轨道类.xlsx",
    CERT_SRC_DIR / "附件1：技术方案（含专项施工方案）编制计划表--轨道类项目.xlsx",
]

def parse_date(val):
    """Excel序列号 or datetime or string → YYYY-MM-DD"""
    if val is None:
        return ''
    if isinstance(val, datetime):
        return val.strftime('%Y-%m-%d')
    if isinstance(val, (int, float)) and val > 40000:
        # Excel serial date (days since 1899-12-30)
        from datetime import timedelta
        return (datetime(1899, 12, 30) + timedelta(days=int(val))).strftime('%Y-%m-%d')
    s = str(val).strip()
    if not s or s in ('/', '-', 'None', 'nan'):
        return ''
    # Try common formats
    for fmt in ['%Y-%m-%d', '%Y-%m-%d %H:%M:%S', '%Y.%m.%d', '%Y/%m/%d']:
        try:
            return datetime.strptime(s[:10], '%Y-%m-%d' if '-' in s[:10] else fmt).strftime('%Y-%m-%d')
        except:
            continue
    # Fallback: just take first 10 chars
    return s[:10] if len(s) >= 10 else s

# ══════════════════════════════════════════
# 表1-专项施工方案（危大方案）
# ══════════════════════════════════════════
print("=" * 60)
print("📖 表1-专项施工方案（3文件合并）...")
print("=" * 60)

all_schemes = []
seen_schemes = set()

for fpath in CERT_FILES:
    if not fpath.exists():
        print(f"  ⚠️ 跳过: {fpath.name}")
        continue
    wb = openpyxl.load_workbook(fpath, data_only=True)
    if '表1-专项施工方案' not in wb.sheetnames:
        print(f"  ⚠️ {fpath.name}: 无 表1-专项施工方案")
        wb.close()
        continue
    ws = wb['表1-专项施工方案']
    file_count = 0
    for row in ws.iter_rows(min_row=3, max_row=ws.max_row, values_only=True):
        region = str(row[1]).strip() if row[1] else ''
        if region != '中东':
            continue
        proj = str(row[3]).strip() if row[3] else ''
        scheme = str(row[6]).strip() if row[6] else ''
        if not scheme or scheme in ('/', '-', 'nan', 'None'):
            continue
        key = (proj, scheme)
        if key in seen_schemes:
            continue
        seen_schemes.add(key)

        all_schemes.append({
            '数据来源': fpath.name,
            '工作表': '表1-专项施工方案',
            '所属区域': region,
            '所属国别': str(row[2]).strip() if row[2] else '',
            '项目名称': proj,
            '方案名称': scheme,
            '编制单位': str(row[7]).strip() if row[7] else '',
            '工程类别': str(row[8]).strip() if row[8] else '',
            '分部工程类别': str(row[9]).strip() if row[9] else '',
            '方案等级': '危大',  # 表1 全部是危大方案
            '是否超一定规模': str(row[11]).strip() if len(row) > 11 and row[11] else '',
            '工程特点/说明': str(row[12]).strip() if len(row) > 12 and row[12] else '',
            '计划开工日期': parse_date(row[13]) if len(row) > 13 else '',
            '方案计划报审日期': parse_date(row[14]) if len(row) > 14 else '',
        })
        file_count += 1
    wb.close()
    print(f"  {fpath.name}: {file_count} 中东条目")

df_schemes = pd.DataFrame(all_schemes)
print(f"\n  表1 中东方案合计: {len(df_schemes)}")
print(f"  超规: {(df_schemes['是否超一定规模']=='是').sum()}, "
      f"一般: {(df_schemes['是否超一定规模']!='是').sum()}")
print(f"  覆盖项目: {df_schemes['项目名称'].nunique()}")

# ══════════════════════════════════════════
# 表2-I II Ⅲ类技术方案
# ══════════════════════════════════════════
print("\n" + "=" * 60)
print("📖 表2-I II Ⅲ类技术方案（3文件合并）...")
print("=" * 60)

all_tech = []
seen_tech = set()

for fpath in CERT_FILES:
    if not fpath.exists():
        continue
    wb = openpyxl.load_workbook(fpath, data_only=True)
    sheet_name = None
    for sn in wb.sheetnames:
        if '表2' in sn and '技术方案' in sn:
            sheet_name = sn
            break
    if not sheet_name:
        print(f"  ⚠️ {fpath.name}: 无 表2 类工作表")
        wb.close()
        continue

    ws = wb[sheet_name]
    file_count = 0
    for row in ws.iter_rows(min_row=3, max_row=ws.max_row, values_only=True):
        region = str(row[1]).strip() if row[1] else ''
        if region != '中东':
            continue
        proj = str(row[3]).strip() if row[3] else ''
        scheme = str(row[6]).strip() if row[6] else ''
        if not scheme or scheme in ('/', '-', 'nan', 'None'):
            continue
        key = (proj, scheme)
        if key in seen_tech:
            continue
        seen_tech.add(key)

        all_tech.append({
            '数据来源': fpath.name,
            '工作表': sheet_name,
            '所属区域': region,
            '所属国别': str(row[2]).strip() if row[2] else '',
            '项目名称': proj,
            '方案名称': scheme,
            '编制单位': str(row[7]).strip() if row[7] else '',
            '工程类别': str(row[8]).strip() if row[8] else '',
            '分部工程类别': '',  # 表2无此列
            '方案等级': str(row[10]).strip() if len(row) > 10 and row[10] else '',  # Ⅰ/Ⅱ/Ⅲ
            '是否超一定规模': '',  # 表2无此列
            '工程特点/说明': str(row[9]).strip() if len(row) > 9 and row[9] else '',
            '计划开工日期': parse_date(row[11]) if len(row) > 11 else '',
            '方案计划报审日期': parse_date(row[13]) if len(row) > 13 else '',
        })
        file_count += 1
    wb.close()
    print(f"  {fpath.name} [{sheet_name}]: {file_count} 中东条目")

df_tech = pd.DataFrame(all_tech)
print(f"\n  表2 中东方案合计: {len(df_tech)}")
print(f"  等级分布: {df_tech['方案等级'].value_counts().to_dict()}")
print(f"  覆盖项目: {df_tech['项目名称'].nunique()}")

# ══════════════════════════════════════════
# 合计统计
# ══════════════════════════════════════════
print("\n" + "=" * 60)
print("📊 汇总统计")
print("=" * 60)
print(f"  表1 危大方案: {len(df_schemes)} 项（超规{(df_schemes['是否超一定规模']=='是').sum()}）")
print(f"  表2 技术方案: {len(df_tech)} 项（Ⅰ/Ⅱ/Ⅲ类）")
print(f"  认定总计:    {len(df_schemes) + len(df_tech)} 项")
all_projects = set(df_schemes['项目名称'].unique()) | set(df_tech['项目名称'].unique())
print(f"  覆盖项目:    {len(all_projects)}")

# ══════════════════════════════════════════
# 项目级汇总（表1 vs OA平台）
# ══════════════════════════════════════════
df_proj = df_schemes.groupby('项目名称').agg(
    认定_危大方案总数=('方案名称', 'count'),
    认定_超规数=('是否超一定规模', lambda x: (x == '是').sum()),
).reset_index()
country_map = df_schemes.groupby('项目名称')['所属国别'].first().to_dict()
df_proj['所属国别'] = df_proj['项目名称'].map(country_map)

PLATFORM_PARQUET = Path("/mnt/y/Openclaw_Hub/03.资源/实施项目 wiki/dashboard/data/2026-05-24/cleaned/methods_cleaned.parquet")
if PLATFORM_PARQUET.exists():
    df_platform = pd.read_parquet(PLATFORM_PARQUET)
    df_platform_valid = df_platform[df_platform['是否有效登记'] == True]
    platform_counts = df_platform_valid.groupby('项目名称').agg(
        平台_方案总数=('方案名称', 'count'),
        平台_超规数=('是否超一定规模', lambda x: (x == '是').sum()),
    ).reset_index()

    matched = []
    for _, cert_row in df_proj.iterrows():
        cert_name = cert_row['项目名称']
        cert_clean = cert_name.replace('工程项目', '').replace('项目', '').strip()
        best_match = None; best_ratio = 0
        for plat_name in platform_counts['项目名称']:
            plat_clean = str(plat_name).replace('工程项目', '').replace('项目', '').strip()
            ratio = difflib.SequenceMatcher(None, cert_clean, plat_clean).ratio()
            if cert_clean in plat_clean or plat_clean in cert_clean:
                ratio = max(ratio, 0.95)
            if ratio > best_ratio:
                best_ratio = ratio; best_match = plat_name
        row_dict = cert_row.to_dict()
        if best_match and best_ratio >= 0.75:
            prow = platform_counts[platform_counts['项目名称'] == best_match].iloc[0]
            row_dict['平台_匹配项目'] = best_match
            row_dict['平台_方案总数'] = int(prow['平台_方案总数'])
            row_dict['平台_超规数'] = int(prow['平台_超规数'])
            row_dict['差额'] = int(prow['平台_方案总数']) - row_dict['认定_危大方案总数']
            row_dict['匹配状态'] = '✅'
        else:
            row_dict['平台_匹配项目'] = ''
            row_dict['平台_方案总数'] = 0
            row_dict['平台_超规数'] = 0
            row_dict['差额'] = -row_dict['认定_危大方案总数']
            row_dict['匹配状态'] = '⚠️ 未匹配'
        matched.append(row_dict)
    df_result = pd.DataFrame(matched)
    print(f"\n  对标: {len(df_result)} 项目, 匹配 {(df_result['匹配状态']=='✅').sum()}")
else:
    df_result = df_proj.copy()
    print("  ⚠️ 未找到平台 Parquet，跳过对标")

# ══════════════════════════════════════════
# 输出
# ══════════════════════════════════════════
# 表1明细
df_schemes.to_csv(OUT_DIR / "certified_schemes_detail.csv", index=False, encoding='utf-8-sig')
df_schemes.to_parquet(OUT_DIR / "certified_schemes_detail.parquet", index=False)
print(f"\n✅ 表1方案明细: {OUT_DIR / 'certified_schemes_detail.csv'} ({len(df_schemes)}行)")

# 表2明细 (NEW)
df_tech.to_csv(OUT_DIR / "certified_tech_schemes_detail.csv", index=False, encoding='utf-8-sig')
df_tech.to_parquet(OUT_DIR / "certified_tech_schemes_detail.parquet", index=False)
print(f"✅ 表2技术方案明细: {OUT_DIR / 'certified_tech_schemes_detail.csv'} ({len(df_tech)}行)")

# 项目级对比
df_result.to_csv(OUT_DIR / "certified_schemes.csv", index=False, encoding='utf-8-sig')
df_result.to_parquet(OUT_DIR / "certified_schemes.parquet", index=False)
print(f"✅ 项目对比: {OUT_DIR / 'certified_schemes.csv'} ({len(df_result)}行)")

# 校验报告
report = {
    "表1_危大方案": len(df_schemes),
    "表1_超规": int((df_schemes['是否超一定规模'] == '是').sum()),
    "表2_技术方案": len(df_tech),
    "表2_等级分布": df_tech['方案等级'].value_counts().to_dict(),
    "认定总计": len(df_schemes) + len(df_tech),
    "覆盖项目": len(all_projects),
    "数据来源": [f.name for f in CERT_FILES],
}
with open(OUT_DIR / "certified_validation.json", 'w', encoding='utf-8') as f:
    json.dump(report, f, ensure_ascii=False, indent=2, default=str)
print(f"✅ 校验报告: {OUT_DIR / 'certified_validation.json'}")
