Commit 87f2a5d7 authored by Nicole Bussola's avatar Nicole Bussola Committed by Alessia Marcolini
Browse files

save train/test metrics

parent 9571af1b
...@@ -36,46 +36,48 @@ from torch.utils.tensorboard import SummaryWriter ...@@ -36,46 +36,48 @@ from torch.utils.tensorboard import SummaryWriter
from networks import CiompiDO, ResNet50_3d from networks import CiompiDO, ResNet50_3d
from dataset import NumpyCSVDataset, augment_3D_HN from dataset import NumpyCSVDataset, augment_3D_HN
from split import train_test_indexes_patient_wise
# In[ ]: # In[ ]:
#os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" #os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
#os.environ["CUDA_VISIBLE_DEVICES"] = "2,3"
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
multigpu = True
# In[ ]: # In[ ]:
DATASET_DIR = f"/thunderdisk/HN/processed/bbox_fixed2_64" #Not augmented but already 64**3 (for faster loading) DATASET_DIR = f"/thunderdisk/HN/processed/bbox_64_augmented_LR"
EXPERIMENT_DIR = f"{PATH}/experiments" EXPERIMENT_DIR = f"{PATH}/experiments"
PRETRAINED_MED3D_WEIGHTS = '/thunderdisk/HN/MedicalNet_pytorch_files/pretrain/resnet_50.pth' PRETRAINED_MED3D_WEIGHTS = '/thunderdisk/HN/MedicalNet_pytorch_files/pretrain/resnet_50.pth'
# ### Settings # ### Settings
# In[ ]: # In[ ]:
EXPERIMENT_NAME = 'Tstage_binary_resnet_pretrained_stack_noTx_' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") EXPERIMENT_NAME = 'LR_noTx_branch_wise_free_aug_CT_' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
settings = { settings = {
'model': ResNet50_3d, 'model': CiompiDO,
'batch_size': 16, 'batch_size': 32,
'lr': 1e-4, 'lr': 1e-5,
'epochs': 100, 'epochs': 80,
'optim': torch.optim.Adam, 'optim': torch.optim.Adam,
'K': 0.2, 'K': 0.2,
'n_classes': 2, #TSTAGE 'n_classes': 2, #TSTAGE
'seed': 1234, 'seed': 1234,
'dropout': 0.5, 'dropout': 0.5,
'size': 64, 'size': 64,
'pretrained': True 'pretrained': 'T-stage',
'weights_mode': 'free', # fine tune all the network or fine tune only the FCH
} }
assert settings['pretrained'] in ['Med3D', 'branch-wise', 'T-stage', '']
os.makedirs(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}', exist_ok=False) os.makedirs(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}', exist_ok=False)
...@@ -93,6 +95,8 @@ SEED = settings['seed'] ...@@ -93,6 +95,8 @@ SEED = settings['seed']
DROPOUT = settings['dropout'] DROPOUT = settings['dropout']
SIZE = settings['size'] SIZE = settings['size']
PRETRAINED = settings['pretrained'] PRETRAINED = settings['pretrained']
WEIGHTS_MODE = settings['weights_mode']
# ### Tensorboard settings # ### Tensorboard settings
...@@ -121,7 +125,7 @@ writer = SummaryWriter(log_dir) ...@@ -121,7 +125,7 @@ writer = SummaryWriter(log_dir)
clinical_data = f'{PATH}/data/clinical_data_noTx.csv' clinical_data = f'{PATH}/data/clinical_data_noTx.csv'
target_column = 'T-stage_binary' target_column = 'Locoregional'
# In[ ]: # In[ ]:
...@@ -129,18 +133,8 @@ target_column = 'T-stage_binary' ...@@ -129,18 +133,8 @@ target_column = 'T-stage_binary'
np.random.seed(SEED) np.random.seed(SEED)
from sklearn.model_selection import StratifiedShuffleSplit dataset = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE , mode='train')
sss = StratifiedShuffleSplit(n_splits=1, test_size=K, random_state=SEED) idx_train, idx_test = train_test_indexes_patient_wise(dataset, test_size=0.2, seed=SEED, stratify=True)
dataset = NumpyCSVDataset(DATASET_DIR , clinical_data , target_column, SIZE , mode="train")
n_samples = len(dataset)
fake_X = np.zeros(n_samples)
labels = dataset.get_labels()
split = sss.split(fake_X, labels)
idx_train, idx_test = next(iter(split))
# Create train-test datasets # Create train-test datasets
...@@ -148,10 +142,10 @@ idx_train, idx_test = next(iter(split)) ...@@ -148,10 +142,10 @@ idx_train, idx_test = next(iter(split))
# In[ ]: # In[ ]:
dataset_test = NumpyCSVDataset(DATASET_DIR , clinical_data , target_column, SIZE, mode="test", augmentation_function=augment_3D_HN) dataset_test = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode="test", augmentation_function=augment_3D_HN)
dataset_test._indexes = idx_test dataset_test._indexes = idx_test
dataset_train = NumpyCSVDataset(DATASET_DIR , clinical_data , target_column, SIZE, mode="train", augmentation_function=augment_3D_HN) dataset_train = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode="train", augmentation_function=augment_3D_HN)
dataset_train._indexes = idx_train dataset_train._indexes = idx_train
...@@ -199,13 +193,17 @@ weights = torch.Tensor(weights).to(device) ...@@ -199,13 +193,17 @@ weights = torch.Tensor(weights).to(device)
# In[ ]: # In[ ]:
model = MODEL(n_classes=N_CLASSES, n_channels=1, modality='CT', dropout=DROPOUT)
model = MODEL(n_classes=N_CLASSES, dropout=DROPOUT) if multigpu:
model = nn.DataParallel(model.to(device)) model = nn.DataParallel(model.to(device))
model = model.module
#print(model.state_dict()['module.conv1.weight'][1,0,0,0]) #print(model.state_dict()['module.conv1.weight'][1,0,0,0])
#model.initialize_weights() #model.initialize_weights()
if PRETRAINED: model.initialize_weights()
if PRETRAINED == 'Med3D':
pretrained_dict = torch.load(PRETRAINED_MED3D_WEIGHTS)['state_dict'] pretrained_dict = torch.load(PRETRAINED_MED3D_WEIGHTS)['state_dict']
model_dict = model.state_dict() model_dict = model.state_dict()
...@@ -217,16 +215,85 @@ if PRETRAINED: ...@@ -217,16 +215,85 @@ if PRETRAINED:
if name in pretrained_dict.keys(): if name in pretrained_dict.keys():
#print(name) #print(name)
model.state_dict()[name].copy_(pretrained_dict[name]) model.state_dict()[name].copy_(pretrained_dict[name])
elif PRETRAINED == 'branch-wise':
pretrained_CT_dict = torch.load(f'{EXPERIMENT_DIR}/Tstage_grouped_noTx_CT_20191025-141049/weights.pth')
pretrained_PT_dict = torch.load(f'{EXPERIMENT_DIR}/Tstage_binary_PET_noTx_20191022-124046/weights.pth')
model_dict = model.state_dict()
pretrained_CT_dict = {k: v for k, v in pretrained_CT_dict.items() if
(k in model_dict) and (model_dict[k].shape == pretrained_CT_dict[k].shape)}
pretrained_PT_dict = {k: v for k, v in pretrained_PT_dict.items() if
(k in model_dict) and (model_dict[k].shape == pretrained_PT_dict[k].shape)}
to_add = 'module.' if multigpu else ''
for name in model.CT_branch.state_dict().keys():
name_complete = to_add + 'CT_branch.' + name
#print(name_complete)
if name_complete in pretrained_CT_dict.keys():
print(name)
model.CT_branch.state_dict()[name].copy_(pretrained_CT_dict[name_complete])
for name in model.PT_branch.state_dict().keys():
name_complete = to_add + 'PT_branch.' + name
#print(name_complete)
if name_complete in pretrained_PT_dict.keys():
print(name)
model.PT_branch.state_dict()[name].copy_(pretrained_PT_dict[name_complete])
elif PRETRAINED == 'T-stage':
pretrained_dict = torch.load(f'{EXPERIMENT_DIR}/Tstage_binary_augmented_noTx_branch_wise_20191025-160304/weights.pth')
model_dict = model.state_dict()
# discard layers not present in destination network or with different shape
pretrained_dict = {k: v for k, v in pretrained_dict.items() if
(k in model_dict) and (model_dict[k].shape == pretrained_dict[k].shape)}
for name in model.state_dict().keys():
if name in pretrained_dict.keys():
#print(name)
model.state_dict()[name].copy_(pretrained_dict[name])
#print(model.state_dict()['module.conv1.weight'][1,0,0,0]) #print(model.state_dict()['module.conv1.weight'][1,0,0,0])
# Optimizer # Optimizer
# In[ ]: # In[ ]:
if WEIGHTS_MODE == 'free':
for param in model.parameters():
param.requires_grad = True
elif WEIGHTS_MODE == 'frozen':
for param in model.features_CT.parameters():
param.requires_grad = False
for param in model.features_PT.parameters():
param.requires_grad = False
# In[21]:
#
# Compute number of trainable parameters
# In[22]:
model_parameters = filter(lambda p: p.requires_grad, model.parameters())
trainable_params = sum([np.prod(p.size()) for p in model_parameters])
total_params = sum([np.prod(p.size()) for p in model.parameters()])
optimizer = torch.optim.Adam(model.parameters(), lr=LR) if WEIGHTS_MODE == 'free':
optimizer = OPTIMIZER(list(model.parameters()), lr = LR)
elif WEIGHTS_MODE == 'frozen':
optimizer = OPTIMIZER(list(model.features.parameters()), lr = LR)
# In[ ]: # In[ ]:
...@@ -369,7 +436,7 @@ with torch.no_grad(): ...@@ -369,7 +436,7 @@ with torch.no_grad():
# trues.append(label) # trues.append(label)
trues_tr.append(dictionary[label]) trues_tr.append(dictionary[label])
probs_tr.append(output.data.cpu().numpy()) probs_tr.append(output.data.cpu().numpy())
filenames_tr.append(data['sample']) filenames_tr.append(data['filename'])
probs_tr = np.concatenate(probs_tr) probs_tr = np.concatenate(probs_tr)
preds_tr = np.concatenate(preds_tr) preds_tr = np.concatenate(preds_tr)
...@@ -381,8 +448,9 @@ ACC_tr = acc(trues_tr, preds_tr) ...@@ -381,8 +448,9 @@ ACC_tr = acc(trues_tr, preds_tr)
prec_tr = precision(trues_tr, preds_tr, average='weighted') prec_tr = precision(trues_tr, preds_tr, average='weighted')
rec_tr = recall(trues_tr, preds_tr, average='weighted') rec_tr = recall(trues_tr, preds_tr, average='weighted')
print("MCC train", MCC_tr, "ACC train", ACC_tr) print("MCC train", round(MCC_tr,3), "ACC train", round(ACC_tr,3))
print("precision train", prec_tr, "recall train", rec_tr ) print('\n')
print("precision train", round(prec_tr,3), "recall train", round(rec_tr,3))
train_metrics = np.array([MCC_tr, ACC_tr, prec_tr, rec_tr]) train_metrics = np.array([MCC_tr, ACC_tr, prec_tr, rec_tr])
...@@ -408,7 +476,7 @@ with torch.no_grad(): ...@@ -408,7 +476,7 @@ with torch.no_grad():
preds_ts.append(pred.data.cpu().numpy()) preds_ts.append(pred.data.cpu().numpy())
trues_ts.append(dictionary[label]) trues_ts.append(dictionary[label])
probs_ts.append(output.data.cpu().numpy()) probs_ts.append(output.data.cpu().numpy())
filenames_ts.append(data['sample']) filenames_ts.append(data['filename'])
probs_ts = np.concatenate(probs_ts) probs_ts = np.concatenate(probs_ts)
preds_ts = np.concatenate(preds_ts) preds_ts = np.concatenate(preds_ts)
...@@ -419,8 +487,9 @@ MCC_ts = mcor(trues_ts, preds_ts) ...@@ -419,8 +487,9 @@ MCC_ts = mcor(trues_ts, preds_ts)
ACC_ts = acc(trues_ts, preds_ts) ACC_ts = acc(trues_ts, preds_ts)
prec_ts = precision(trues_ts, preds_ts, average='weighted') prec_ts = precision(trues_ts, preds_ts, average='weighted')
rec_ts = recall(trues_ts, preds_ts, average='weighted') rec_ts = recall(trues_ts, preds_ts, average='weighted')
print("MCC test", MCC_ts, "ACC test", ACC_ts) print("MCC test", round(MCC_ts,3), "ACC test", round(ACC_ts,3))
print("precision test", prec_ts, "recall test", rec_ts ) print('\n')
print("precision test", round(prec_ts,3), "recall test", round(rec_ts,3))
test_metrics = np.array([MCC_ts, ACC_ts, prec_ts, rec_ts]) test_metrics = np.array([MCC_ts, ACC_ts, prec_ts, rec_ts])
...@@ -482,7 +551,7 @@ np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/filenames_ts.npy', filenames_ts) ...@@ -482,7 +551,7 @@ np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/filenames_ts.npy', filenames_ts)
# In[ ]: # In[ ]:
metrics_out = np.stack([train_metrics, test_metrics], 0) metrics_out = pd.DataFrame([train_metrics, test_metrics], columns=['MCC','ACC', 'PREC', 'REC'], index=['train','test']) #np.stack([train_metrics, test_metrics], 0)
np.savetxt(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/metrics_out.txt', metrics_out) np.savetxt(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/metrics_out.txt', metrics_out)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment