#!/usr/bin/env python3
"""
B1 数据清洗：危大方案编审进度
输入: 技术方案统计表.xlsx (原始)
输出: methods_cleaned.parquet (33列·中东区域)
参照: 2026-05-04_技术方案统计表 Rev01 FNL.xlsx → Sheet '统计表-清洗后'
"""
import pandas as pd
import openpyxl
from datetime import datetime, date
from pathlib import Path
import json
import warnings
warnings.filterwarnings('ignore')

# ============================================================
# CONFIG
# ============================================================
DATA_DATE = "2026-05-24"
RAW_DIR = Path(f"/mnt/y/Openclaw_Hub/03.资源/实施项目 wiki/raw/月报数据/{DATA_DATE}")
OUT_DIR = Path(f"/mnt/y/Openclaw_Hub/03.资源/实施项目 wiki/dashboard/data/{DATA_DATE}/cleaned")
OUT_DIR.mkdir(parents=True, exist_ok=True)

INPUT_FILE = RAW_DIR / "技术方案统计表.xlsx"
TODAY = date.today()

# ============================================================
# Step 1: 读取原始数据
# ============================================================
print("📖 读取原始数据...")
wb = openpyxl.load_workbook(INPUT_FILE, data_only=True)
ws = wb['sheet1']  # 原始数据在 sheet1

# 提取表头 (Row 2)
headers = {}
for cell in ws[2]:
    if cell.value:
        headers[cell.column_letter] = str(cell.value).strip()

print(f"  表头列: {len(headers)} 列")
for k, v in sorted(headers.items()):
    print(f"    [{k}] {v}")

# 提取数据行 (Row 3+)
raw_rows = []
for row in ws.iter_rows(min_row=3, max_row=ws.max_row, values_only=False):
    vals = {}
    for cell in row:
        try:
            cl = cell.column_letter
            if cl in headers:
                vals[headers[cl]] = cell.value
        except AttributeError:
            pass  # MergedCell
    # 跳过空行
    if any(v is not None for v in vals.values()):
        raw_rows.append(vals)

wb.close()
print(f"  原始行数: {len(raw_rows)}")

# ============================================================
# Step 2: 转为 DataFrame
# ============================================================
df = pd.DataFrame(raw_rows)
print(f"\n📊 原始列: {list(df.columns)}")
print(f"  总行数: {len(df)}")

# ============================================================
# Step 3: Forward-fill 区域/国别/项目名称
# ============================================================
# 项目名称 forward-fill（同一项目的方案行共享项目名）
proj_col = '项目名称'
if proj_col in df.columns:
    before = df[proj_col].notna().sum()
    df[proj_col] = df[proj_col].fillna(method='ffill')
    after = df[proj_col].notna().sum()
    print(f"🔧 Forward-fill 项目名称: {before} → {after}")

# 产业链 forward-fill + 清洗异常值（按项目分组，防止跨项目串位）
chain_col = '产业链'
if chain_col in df.columns:
    before = df[chain_col].notna().sum()
    # 将纯数字值 (如 10, 5, 0) 替换为 NaN
    df[chain_col] = df[chain_col].apply(
        lambda x: None if pd.api.types.is_number(x) or (isinstance(x, str) and x.strip().isdigit()) else x
    )
    # 按项目分组 forward-fill（防止工商银行→道路维护 串位为"建筑"）
    df[chain_col] = df.groupby(proj_col)[chain_col].transform(lambda x: x.fillna(method='ffill'))
    after = df[chain_col].notna().sum()
    
    # Step 4.5a: CSV 查表修正（优先级高于项目名推断）
    PROJECTS_FULL_CSV = RAW_DIR.parent.parent / "projects_full_2026-05-20.csv"
    csv_fixed = 0
    if PROJECTS_FULL_CSV.exists():
        import csv as csv_module
        chain_lookup = {}
        with open(PROJECTS_FULL_CSV, 'r', encoding='utf-8-sig') as f:
            for row in csv_module.DictReader(f):
                name = row.get('项目名称(中文)', '')
                chain = row.get('产业链(五全十链)', '')
                if name and chain and chain.strip():
                    chain_lookup[name.strip()] = chain.strip()
        
        for idx in df[df[chain_col].isna()].index:
            proj = str(df.at[idx, proj_col]).strip()
            if proj in chain_lookup:
                df.at[idx, chain_col] = chain_lookup[proj]
                csv_fixed += 1
        
        if csv_fixed > 0:
            print(f"📖 CSV查表补产业链: {csv_fixed} 条")
    
    # 对分组填充 + CSV 查表后仍为空的，从项目名推断
    after_csv = df[chain_col].notna().sum()
    CHAIN_KEYWORDS = {
        '道桥': ['道路', '桥梁', '桥', '公路', '高速', '交通', '立交', '铁路', '轨道'],
        '建筑': ['房建', '建筑', '大楼', '酒店', '办公', '住宅', '别墅', '公寓', '商场'],
        '管网': ['供水', '排水', '污水', '管网', '管线', '基础设施', '雨水'],
        '港口': ['港口', '码头', '水工', '海工', '疏浚', '吹填', '航道', '船'],
        '电力': ['电力', '发电', '光伏', '风电', '变电站'],
        '生态环保': ['环保', '生态', '绿化', '园林'],
    }
    
    def infer_chain_from_name(name):
        if pd.isna(name):
            return None
        name = str(name)
        for chain, keywords in CHAIN_KEYWORDS.items():
            for kw in keywords:
                if kw in name:
                    return chain
        return None
    
    still_nan = df[chain_col].isna()
    if still_nan.any():
        for idx in df[still_nan].index:
            inferred = infer_chain_from_name(df.at[idx, proj_col])
            if inferred:
                df.at[idx, chain_col] = inferred
        after2 = df[chain_col].notna().sum()
        print(f"🔧 Forward-fill 产业链（按项目）: {before} → {after} → {after2} (清洗数字值 + 项目名推断)")

# '所属区域' 列可能在 B 列，父行有值子行为空
region_col = '所属区域'
if region_col in df.columns:
    before = df[region_col].notna().sum()
    df[region_col] = df[region_col].fillna(method='ffill')
    after = df[region_col].notna().sum()
    print(f"\n🔧 Forward-fill 区域列: {before} → {after}")

# '所属国别' 同样 forward-fill
country_col = '所属国别'
if country_col in df.columns:
    before = df[country_col].notna().sum()
    df[country_col] = df[country_col].fillna(method='ffill')
    after = df[country_col].notna().sum()
    print(f"🔧 Forward-fill 国别列: {before} → {after}")

# 修正 forward-fill 导致的国别串位：从项目名推断国别
COUNTRY_KEYWORDS = {
    '阿联酋': '阿拉伯联合酋长国',
    '沙特': '沙特阿拉伯',
    '卡塔尔': '卡塔尔',
    '科威特': '科威特',
    '埃及': '埃及',
    '阿曼': '阿曼',
    '苏丹': '苏丹',
    '巴林': '巴林',
    '约旦': '约旦',
    '伊拉克': '伊拉克',
    '黎巴嫩': '黎巴嫩',
    '叙利亚': '叙利亚',
}

def infer_country_from_project(proj_name):
    """从项目名称推断国别"""
    if pd.isna(proj_name):
        return None
    proj_name = str(proj_name)
    for kw, country in COUNTRY_KEYWORDS.items():
        if kw in proj_name:
            return country
    return None

# 对每行验证：如果项目名包含的国别关键词与 forward-filled 国别不同，修正
if country_col in df.columns and '项目名称' in df.columns:
    fixed_count = 0
    for idx in df.index:
        proj = df.at[idx, '项目名称']
        current_country = df.at[idx, country_col]
        inferred = infer_country_from_project(proj)
        if inferred and str(current_country) != inferred:
            df.at[idx, country_col] = inferred
            fixed_count += 1
    print(f"🔧 修正国别串位: {fixed_count} 条（从项目名推断修正）")

    # 对 forward-fill 后仍是 NaN 的，也尝试推断
    nan_before = df[country_col].isna().sum()
    for idx in df[df[country_col].isna()].index:
        proj = df.at[idx, '项目名称']
        inferred = infer_country_from_project(proj)
        if inferred:
            df.at[idx, country_col] = inferred
    nan_after = df[country_col].isna().sum()
    if nan_before > nan_after:
        print(f"🔧 补全国别: {nan_before} → {nan_after}（从项目名推断）")

# ============================================================
# Step 4.5: 产业链查表修正（projects_full CSV 为权威数据）
# ============================================================
# 原始表中产业链列存在数值编码（如"10""09"）等非标准值，需查表修正
PROJECTS_FULL_CSV = RAW_DIR.parent / "projects_full_2026-05-20.csv"
if PROJECTS_FULL_CSV.exists():
    import csv
    chain_lookup = {}
    with open(PROJECTS_FULL_CSV, 'r', encoding='utf-8-sig') as f:
        for row in csv.DictReader(f):
            name = row.get('项目名称(中文)', '')
            chain = row.get('产业链(五全十链)', '')
            if name and chain and chain.strip():
                chain_lookup[name.strip()] = chain.strip()
    print(f"📖 产业链查表: {len(chain_lookup)} 个项目")
    
    # 修正中东区域 DataFrame 中的产业链
    fixed_chain = 0
    for idx in df_me.index:
        proj = str(df_me.at[idx, '项目名称']).strip()
        current = str(df_me.at[idx, '产业链']).strip()
        # 判断是否为非标准值（纯数字、空、nan）
        is_nonstandard = (
            current in ('', 'nan', 'None') or
            current.replace('.', '').isdigit()  # 如 "10", "09"
        )
        if is_nonstandard and proj in chain_lookup:
            new_val = chain_lookup[proj]
            if new_val != current:
                df_me.at[idx, '产业链'] = new_val
                fixed_chain += 1
    
    print(f"🔧 产业链查表修正: {fixed_chain} 条")
else:
    print(f"⚠️ 未找到 projects_full CSV: {PROJECTS_FULL_CSV}")

# ============================================================
# Step 4: 过滤中东区域
# ============================================================
print(f"\n🔍 区域分布:")
if region_col in df.columns:
    region_counts = df[region_col].value_counts()
    for region, count in region_counts.head(10).items():
        print(f"  {region}: {count}")

# 过滤中东
ME_REGION = '中国港湾中东区域公司'
if region_col in df.columns:
    df_me = df[df[region_col] == ME_REGION].copy()
else:
    # fallback: try 所属区域 column
    for col in df.columns:
        if '中东' in str(col) or '区域' in str(col):
            me_mask = df[col].astype(str).str.contains('中东', na=False)
            if me_mask.sum() > 0:
                df_me = df[me_mask].copy()
                print(f"  使用列 '{col}' 过滤中东: {len(df_me)} 行")
                break
    else:
        df_me = df.copy()
        print("  ⚠️ 未找到区域列，保留全量数据")

print(f"  中东区域行数: {len(df_me)}")

# ============================================================
# Step 5: 派生计算列（33列目标）
# ============================================================
print(f"\n🔧 派生计算列...")

# 5.1 解析日期
date_cols_original = ['分部分项工程计划开工日期']
for col in date_cols_original:
    if col in df_me.columns:
        df_me[col] = pd.to_datetime(df_me[col], errors='coerce')

# 5.2 方案计划报审/备案日期 = 开工日期 - 15天
if '分部分项工程计划开工日期' in df_me.columns:
    df_me['方案计划报审/备案日期'] = df_me['分部分项工程计划开工日期'] - pd.Timedelta(days=15)

# 5.3 方案状态（原始在 W 列，映射为 '方案状态'）
# 检查是否有 '方案状态' 列，或从原始表 W 列读取
status_col = None
for col in df_me.columns:
    if '状态' in str(col):
        status_col = col
        break

if status_col:
    print(f"  方案状态列: '{status_col}'")
    df_me['方案状态'] = df_me[status_col].astype(str)
else:
    df_me['方案状态'] = '未知'

# 5.4 开工年份/月份
if '分部分项工程计划开工日期' in df_me.columns:
    df_me['开工年份'] = df_me['分部分项工程计划开工日期'].dt.year
    df_me['开工月份'] = df_me['分部分项工程计划开工日期'].dt.month

# 5.5 计划开工日期倒数
if '分部分项工程计划开工日期' in df_me.columns:
    df_me['计划开工日期倒数'] = (df_me['分部分项工程计划开工日期'] - pd.Timestamp(TODAY)).dt.days

# 5.6 是否完成审批
if status_col:
    approved_keywords = ['已审批', '已备案']
    df_me['是否完成审批'] = df_me['方案状态'].apply(
        lambda x: any(kw in str(x) for kw in approved_keywords)
    )
else:
    df_me['是否完成审批'] = False

# 5.7 是否超一定规模
oversize_col = '是否超一定规模'
if oversize_col in df_me.columns:
    # Forward-fill to fix merged cells (parent has value, child rows NaN)
    before = df_me[oversize_col].notna().sum()
    df_me[oversize_col] = df_me[oversize_col].fillna(method='ffill')
    after = df_me[oversize_col].notna().sum()
    print(f"🔧 Forward-fill 是否超一定规模: {before} → {after}")
    df_me[oversize_col] = df_me[oversize_col].astype(str)
else:
    df_me[oversize_col] = '否'

# 5.8 是否有效登记
# 已作废 → 无效；重复方案 → 去重标记
def is_valid_registration(row):
    status = str(row.get('方案状态', ''))
    # 已作废 → 无效
    if '已作废' in status:
        return False
    # 方案名称含"作废"/"重复" → 无效
    name = str(row.get('方案名称', ''))
    if '作废' in name or '拟作废' in name:
        return False
    return True

df_me['是否有效登记'] = df_me.apply(is_valid_registration, axis=1)

valid_count = df_me['是否有效登记'].sum()
invalid_count = len(df_me) - valid_count
print(f"  有效登记: {valid_count} | 无效登记(已作废): {invalid_count}")

# 5.9 是否逾期未审批
if '分部分项工程计划开工日期' in df_me.columns:
    df_me['是否逾期未审批'] = (
        (df_me['分部分项工程计划开工日期'] < pd.Timestamp(TODAY)) &
        (~df_me['是否完成审批'])
    )
else:
    df_me['是否逾期未审批'] = False

# 5.10 预警信号
def calc_warning(row):
    status = str(row.get('方案状态', ''))
    days_to_start = row.get('计划开工日期倒数', 999)
    is_approved = row.get('是否完成审批', False)
    is_valid = row.get('是否有效登记', True)
    
    if not is_valid:
        return '无预警信号'
    if is_approved:
        return '无预警信号'
    
    # 🔴 红色：在实施但未审批
    if '在实施' in status and ('未审批' in status or '审批中' in status):
        return '🔴 红色预警'
    
    # 未实施
    if days_to_start is None or pd.isna(days_to_start):
        return '无预警信号'
    
    if days_to_start <= 30 and days_to_start >= 0:
        return '🟠 橙色预警'
    elif days_to_start <= 45 and days_to_start > 30:
        return '🟡 黄色预警'
    
    return '无预警信号'

df_me['预警信号'] = df_me.apply(calc_warning, axis=1)

# 5.11 审批进行中
df_me['审批进行中'] = df_me['方案状态'].apply(
    lambda x: '在实施' in str(x) or '审批中' in str(x)
)

# 5.12 方案报审滞后时间 + 报审次数（原表值，保持）
df_me['方案报审滞后时间'] = pd.to_numeric(df_me['方案报审滞后时间'], errors='coerce')
df_me['报审次数'] = 1
# 原因备注
df_me['原因备注'] = df_me.get('原因备注', '')

# ============================================================
# Step 5B: 从原始 W 列提取方案状态（如未被映射）
# ============================================================
# 原始技术方案统计表.xlsx 的 W 列可能是方案状态
# 如果上面没有正确映射，从原始读取
if df_me['方案状态'].eq('未知').all() or df_me['方案状态'].eq('nan').all():
    print("  ⚠️ 方案状态未映射，重新从原始表读取...")
    wb2 = openpyxl.load_workbook(INPUT_FILE, data_only=True)
    ws2 = wb2['sheet1']
    
    # 读 W 列（方案状态）
    status_map = {}
    for row in ws2.iter_rows(min_row=3, max_row=ws2.max_row, values_only=False):
        seq = row[0].value  # A列=序号
        w_val = row[22].value if len(row) > 22 else None  # W列=第23列(0-indexed)
        if seq is not None and w_val is not None:
            status_map[int(seq)] = str(w_val).strip()
    wb2.close()
    
    # 按序号映射
    if '序号' in df_me.columns:
        df_me['方案状态'] = df_me['序号'].map(status_map).fillna('未知')
        print(f"  映射了 {sum(df_me['方案状态'] != '未知')} 条状态")
    
    # 重新计算依赖状态的计算列
    approved_keywords = ['已审批', '已备案']
    df_me['是否完成审批'] = df_me['方案状态'].apply(
        lambda x: any(kw in str(x) for kw in approved_keywords)
    )
    df_me['预警信号'] = df_me.apply(calc_warning, axis=1)
    df_me['审批进行中'] = df_me['方案状态'].apply(
        lambda x: '在实施' in str(x) or '审批中' in str(x)
    )
    if '分部分项工程计划开工日期' in df_me.columns:
        df_me['是否逾期未审批'] = (
            (df_me['分部分项工程计划开工日期'] < pd.Timestamp(TODAY)) &
            (~df_me['是否完成审批'])
        )

# ============================================================
# Step 6: 汇总统计
# ============================================================
print(f"\n📊 汇总统计:")
total = len(df_me)
oversized = (df_me['是否超一定规模'] == '是').sum()
approved = df_me['是否完成审批'].sum()
warnings_count = (df_me['预警信号'] != '无预警信号').sum()
red_warnings = (df_me['预警信号'] == '🔴 红色预警').sum()
orange_warnings = (df_me['预警信号'] == '🟠 橙色预警').sum()
unapproved = total - approved
countries = df_me['所属国别'].nunique() if '所属国别' in df_me.columns else 0

print(f"  有效方案总数: {total}")
print(f"  超一定规模: {oversized} ({oversized/total*100:.0f}%)")
print(f"  已审批: {approved} ({approved/total*100:.0f}%)")
print(f"  未审批: {unapproved}")
print(f"  预警信号: {warnings_count} (🔴{red_warnings} 🟠{orange_warnings})")
print(f"  覆盖国别: {countries}")

# 状态分布
if '方案状态' in df_me.columns:
    print(f"\n  方案状态分布:")
    for status, count in df_me['方案状态'].value_counts().head(10).items():
        print(f"    {status}: {count}")

# ============================================================
# Step 7: 输出
# ============================================================
# 7.1 保存 Parquet
PARQUET_PATH = OUT_DIR / "methods_cleaned.parquet"
df_me.to_parquet(PARQUET_PATH, index=False)
print(f"\n✅ 清洗后数据: {PARQUET_PATH} ({len(df_me)} 行 × {len(df_me.columns)} 列)")

# 7.2 保存 CSV (方便人工审核)
CSV_PATH = OUT_DIR / "methods_cleaned.csv"
df_me.to_csv(CSV_PATH, index=False, encoding='utf-8-sig')
print(f"✅ CSV审核文件: {CSV_PATH}")

# 7.3 校验报告
report = {
    "date": DATA_DATE,
    "source": str(INPUT_FILE),
    "total_rows_raw": len(df),
    "me_rows": len(df_me),
    "summary": {
        "有效方案总数": total,
        "超一定规模": int(oversized),
        "超规占比": f"{oversized/total*100:.0f}%",
        "已审批": int(approved),
        "审批率": f"{approved/total*100:.0f}%",
        "未审批": int(unapproved),
        "预警信号总数": int(warnings_count),
        "红色预警": int(red_warnings),
        "橙色预警": int(orange_warnings),
        "覆盖国别": int(countries),
    },
    "columns": list(df_me.columns),
    "column_count": len(df_me.columns),
    "warnings": [],
    "anomalies": []
}

# 检查异常
if '分部分项工程计划开工日期' in df_me.columns:
    overdue_unapproved = df_me[
        (df_me['分部分项工程计划开工日期'] < pd.Timestamp(TODAY)) &
        (~df_me['是否完成审批'])
    ]
    if len(overdue_unapproved) > 0:
        for _, row in overdue_unapproved.iterrows():
            report['anomalies'].append({
                "type": "逾期未审批",
                "项目": str(row.get('项目名称', '')),
                "方案": str(row.get('方案名称', '')),
                "开工日期": str(row.get('分部分项工程计划开工日期', ''))[:10],
                "状态": str(row.get('方案状态', '')),
            })

REPORT_PATH = OUT_DIR / "validation_report.json"
with open(REPORT_PATH, 'w', encoding='utf-8') as f:
    json.dump(report, f, ensure_ascii=False, indent=2, default=str)
print(f"✅ 校验报告: {REPORT_PATH}")

# 7.4 打印摘要
print(f"\n{'='*60}")
print(f"📋 校验报告摘要")
print(f"{'='*60}")
for k, v in report['summary'].items():
    print(f"  {k}: {v}")
if report['anomalies']:
    print(f"\n  ⚠️ 异常项: {len(report['anomalies'])} 条")
    for a in report['anomalies'][:5]:
        print(f"    - {a['type']}: {a['项目']} / {a['方案'][:40]}")
print(f"\n  📂 输出目录: {OUT_DIR}")
print(f"  📄 {PARQUET_PATH.name}")
print(f"  📄 {CSV_PATH.name}")
print(f"  📄 {REPORT_PATH.name}")
