else.py

import open_clip_custom as conch_clip
import torch
import os
import random
import numpy as np
import matplotlib.pyplot as plt
import torch.nn as nn
from torch.optim import lr_scheduler
import torch.optim as optim
import pandas as pd
from transformers import AutoModel, AutoTokenizer
from torchvision import transforms
from PIL import Image
import json
import shutil
import h5py


ckpt_path = "C:\\Users\\Administrator\\Code\\pathology\\model\\conch\\pytorch_model.bin"
device = "cuda" if torch.cuda.is_available() else "cpu"
SEED = 2025
torch.manual_seed(SEED)

def load_conch(model_path, device):
    model, processor = conch_clip.create_model_from_pretrained("conch_ViT-B-16", checkpoint_path=model_path)
    model.to(device)
    return model, processor

    

def plot_lr():
    # 使用 nn.Sequential 定义一个简单的线性模型
    model = nn.Sequential(
        nn.Linear(1, 1)
    )

    # 初始化学习率和其他参数
    lr = 1e-4
    epoch = 30

    # 设置优化器和调度器
    warm_up_iter = int(epoch * 0.2)
    T_max = epoch

    lambda0 = lambda cur_iter: cur_iter / warm_up_iter if cur_iter < warm_up_iter else 0.5 * (1.0 + np.cos(np.pi * ((cur_iter - warm_up_iter) / (T_max - warm_up_iter))))

    optimizer = optim.Adam(model.parameters(), lr=lr)
    scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda0)

    # 记录每个迭代的学习率
    lrs = []
    total_iterations = epoch  # 假设有100个batch

    for cur_iter in range(total_iterations):
        optimizer.zero_grad()
        # 前向传播、反向传播、优化步骤
        output = model(torch.randn(1))  # 随机输入
        loss = output.sum()  # 随机损失
        loss.backward()
        optimizer.step()
        scheduler.step()
        lrs.append(optimizer.param_groups[0]['lr'])

    # 绘制学习率变化曲线
    plt.figure(figsize=(10, 6))
    plt.plot(lrs, label='Learning Rate')
    plt.xlabel('Iterations')
    plt.ylabel('Learning Rate')
    plt.title('Learning Rate Scheduler')
    plt.legend()
    plt.grid(True)
    plt.savefig('learning_rate.png')  # 保存图像
    plt.show()


# #[normal, b, is, iv] -> [0, 1, 2, 3]
# def divide_Bach_dataset(tif_dir, keep_path, save_path='/ailab/group/pjlab-medai/zhouxiao/pathology/features/keep/bach'):


# #[lung_aca, lung_n, lung_scc, colon_aca, colon_n] -> [0, 1, 2, 3, 4]
# def divide_lc25000_dataset(jpeg_dir='/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/Lc25000/images_224', csv_path='/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/Lc25000/Lc25000_test.csv', 
#                            keep_path='/ailab/group/pjlab-medai/zhouxiao/pathology/model/keep', save_path='/ailab/group/pjlab-medai/zhouxiao/pathology/features/keep/lc25000'): 

    
# #[blood, cancer, normal, other, stroma] -> [0, 1, 2, 3, 4]
# def divide_renalcell_dataset(jpeg_dir='/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/Renalcell/images_224', csv_path='/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/Renalcell/Renalcell_test.csv', 
#                            keep_path='/ailab/group/pjlab-medai/zhouxiao/pathology/model/keep', save_path='/ailab/group/pjlab-medai/zhouxiao/pathology/features/keep/renalcell'): 


# #[necrosis, skeletal, sweatglands, vessel, elastosis, chondraltissue, hairfollicle, epidermis, nerves, subcutis, dermis, sebaceousglands, sqcc, melanoma, bcc, naevus]
# def divide_skincancer_dataset(jpeg_dir='/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/Skincancer/images_224', csv_path='/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/Skincancer/Skincancer_test.csv', 
#                            keep_path='/ailab/group/pjlab-medai/zhouxiao/pathology/model/keep', save_path='/ailab/group/pjlab-medai/zhouxiao/pathology/features/keep/skincancer'): 


def write_train_IDs(json_path='/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/testset_division.json'):
    train_IDs = {}
    with open(json_path, 'r', encoding='utf-8') as f:
        meta = json.load(f)
    for dataset in meta.keys():
        print(f'###################{dataset}###########################')
        train_IDs[dataset] = {}
        csv_path = f'/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/{dataset}/{dataset}_test.csv'
        img_dir = f'/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/{dataset}/images_224'
        label_df = pd.read_csv(csv_path, sep='\t')
        info = meta[dataset]
        test_ids = info['test_IDs']
        name2label = info['name2label']
        for label in name2label.values():
            train_IDs[dataset][label] = []
        file_names = os.listdir(img_dir)
        for file in file_names:
            slide_ID = os.path.splitext(file)[0]
            if slide_ID in test_ids:
                print(slide_ID + ' exists in testset')
            else:
                row = label_df[label_df['image_name'] == file]
                label = name2label[row.iloc[0]['label']]
                train_IDs[dataset][label].append(slide_ID)
                print(f'{slide_ID} label: {label}')
    with open('/ailab/group/pjlab-medai/zhouxiao/pathology/data/patch_level_data/patch_classification/trainset_division.json', 'w', encoding='utf-8') as f:
        json.dump(train_IDs, f, indent=4)
    return


def copy_normal_images(source_root, target_root):
    if not os.path.exists(target_root):
        os.makedirs(target_root)
    # 遍历所有次级目录
    for subdir in os.listdir(source_root):
        source_subdir = os.path.join(source_root, subdir)
        if not os.path.isdir(source_subdir):
            continue  # 跳过非目录文件

        # 处理 normal_images
        source_normal = os.path.join(source_subdir, "normal_images")
        if os.path.isdir(source_normal):
            # 创建目标目录
            target_normal = os.path.join(target_root, subdir, "normal_images")
            os.makedirs(target_normal, exist_ok=True)

            # 获取所有文件（按需过滤扩展名）
            all_files = [f for f in os.listdir(source_normal) 
                        if os.path.isfile(os.path.join(source_normal, f))]
            if all_files:
                # 计算复制数量（至少 1 个文件）
                num_copy = max(1, int(len(all_files) * 0.1))
                files_to_copy = random.sample(all_files, num_copy)

                # 复制文件
                for file in files_to_copy:
                    src = os.path.join(source_normal, file)
                    dst = os.path.join(target_normal, file)
                    shutil.copy2(src, dst)  # 使用copy2保留元数据
                    print(f"复制 normal 文件: {src} → {dst}")

        # 处理 abnormal_images
        source_abnormal = os.path.join(source_subdir, "abnormal_images")
        if os.path.isdir(source_abnormal):
            # 创建目标目录结构
            target_abnormal = os.path.join(target_root, subdir, "abnormal_images")
            os.makedirs(os.path.dirname(target_abnormal), exist_ok=True)

            # 复制整个目录（Python 3.8+需要dirs_exist_ok参数）
            if os.path.exists(target_abnormal):
                # 如果目标已存在，使用copytree合并（需要Python 3.8+）
                shutil.copytree(
                    source_abnormal,
                    target_abnormal,
                    dirs_exist_ok=True,
                    copy_function=shutil.copy2
                )
            else:
                # 如果目标不存在，直接复制
                shutil.copytree(source_abnormal, target_abnormal, copy_function=shutil.copy2)
            
            print(f"复制 abnormal 目录: {source_abnormal} → {target_abnormal}")
            


def divide_brca(): # Warning:train test写反了
    json_path='/ailab/group/pjlab-medai/zhouxiao/pathology/data/tcga_WSI_data/dataset_division.json'
    if os.path.exists(json_path):
        with open(json_path, 'r', encoding='utf-8') as f:
            meta = json.load(f)
    else:
        meta = {}
    meta['BRCA'] = {'train_IDs':{1:[], 2:[]}, 'test_IDs':{1:[], 2:[]}, 'name2label':{'Invasive Ductal Carcinoma':1, 'Invasive Lobular Carcinoma':2}}
    anno_dir = '/ailab/group/pjlab-medai/zhouxiao/pathology/annotations/tcga/BRCA'
    test = []
    for i, name in enumerate(['IDC', 'ILC']):
        label = i+1
        anno_path = os.path.join(anno_dir, name) + '_Anno'
        test_files = random.sample(os.listdir(anno_path), 15)
        test_IDs = [os.path.splitext(file)[0] for file in test_files]
        meta['BRCA']['train_IDs'][label] = test_IDs
        test.extend(test_IDs)
    
    all_dir = '/ailab/group/pjlab-medai/zhouxiao/pathology/features/keep/wsi_features/tcga/BRCA/h5_files'
    label_csv = "/ailab/group/pjlab-medai/zhouxiao/pathology/features/keep/wsi_features/tcga/BRCA/tcga_brca_test.csv"
    label_df = pd.read_csv(label_csv)
    for file in os.listdir(all_dir):
        slide_ID = os.path.splitext(file)[0]
        row = label_df[label_df['slide_id'] == slide_ID]
        if not row.empty:
            diagnosis = row.iloc[0]['Diagnosis']
            label = meta['BRCA']['name2label'][diagnosis]
            meta['BRCA']['test_IDs'][label].append(slide_ID)
        else:
            raise ValueError(f"Label of {slide_ID} not found in the CSV file.")
            continue  # Skip this h5 file if no corresponding row is found in the CSV file
        
    with open(json_path, 'w', encoding='utf-8') as f:
        json.dump(meta, f, indent=4)
        
    
                
if __name__ == "__main__":
    divide_brca()
    # path = 'D:\\HDX\\SJTU\\AI4Med\\pathology\\myCoOp\\TCGA-5L-AAT1-01Z-00-DX1.F3449A5B-2AC4-4ED7-BF44-4C8946CDB47D.h5'
    # with h5py.File(path, 'r') as f:
    #     coords = f['coords'][:]
    #     features = f['features'][:]
    # asset_dict = {'features': features, 'coords': coords}
    # something = np.random.randint(0, 100, (100, 3))
    # asset_dict['something'] = something
    # save_hdf5('test.h5', asset_dict)
    # name = '000001.jpg'
    # index = int(os.path.splitext(name)[0])-1
    # print(index)
    
    # import pandas as pd

    # # 文件路径
    # file_path = 'C:\\Users\\Administrator\\Code\\pathology\\scripts\\myCoOp\\keep\\Bach_test.csv'  # 替换为你的文件路径

    # # 读取 CSV 文件，分隔符为 \t
    # df = pd.read_csv(file_path, sep='\t')

    # # 获取第一行
    # first_row = df.iloc[0].copy()  # 使用 copy() 避免修改原始数据

    # # 修改 image_name 列中的 n002.tif 为 n001.tif
    # first_row['image_name'] = first_row['image_name'].replace('n002.tif', 'n001.tif')

    # # 将修改后的第一行追加到最后一行
    # df = pd.concat([df, first_row.to_frame().T], ignore_index=True)

    # # 保存修改后的文件，分隔符为 \t
    # new_file_path = file_path.replace('.csv', '_modified.csv')  # 新文件名
    # df.to_csv(new_file_path, sep='\t', index=False)

    # print(f"文件已保存为: {new_file_path}")