trainingTstage.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training network for featture extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "PATH = os.getcwd()\n",
    "print(PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import gc\n",
    "import pickle\n",
    "import sys\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import matthews_corrcoef as mcor, accuracy_score as acc, recall_score as recall, precision_score as precision, confusion_matrix\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "from networks import CiompiDO, ResNet50_3d\n",
    "from dataset import NumpyCSVDataset, augment_3D_HN \n",
    "from split import train_test_indexes_patient_wise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "multigpu = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_DIR = f\"/thunderdisk/HN/processed/bbox_fixed2_64/\" #Not augmented but already 64**3 (for faster loading)\n",
    "EXPERIMENT_DIR = f\"{PATH}/experiments\"\n",
    "\n",
    "PRETRAINED_MED3D_WEIGHTS = '/thunderdisk/HN/MedicalNet_pytorch_files/pretrain/resnet_50.pth'\n",
    "PRETRAINED_T_STAGE = f'{EXPERIMENT_DIR}/Tstage_binary_augmented_noTx_branch_wise_20191028-104101/checkpoint_40.pth'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EXPERIMENT_NAME = 'Tstage_grouped_noTx_CT_valieres_' + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "\n",
    "settings = {\n",
    "            'model': CiompiDO,\n",
    "            'batch_size': 32,\n",
    "            'lr': 1e-5,\n",
    "            'epochs': 300,\n",
    "            'optim': torch.optim.Adam,\n",
    "            'K': 0.2,\n",
    "            'n_classes': 2, #TSTAGE\n",
    "            'seed': 1234,\n",
    "            'dropout': 0.5,\n",
    "            'split': 'valieres',\n",
    "            'size': 64,\n",
    "            'pretrained': '',\n",
    "            }\n",
    "\n",
    "assert settings['split'] in ['valeries', '8020']\n",
    "assert settings['pretrained'] in ['Med3D', 'branch-wise', 'T-stage', '']\n",
    "\n",
    "os.makedirs(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}', exist_ok=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL = settings['model']\n",
    "BATCH_SIZE = settings['batch_size']\n",
    "LR = settings['lr']\n",
    "EPOCHS = settings['epochs']\n",
    "OPTIMIZER = settings['optim']\n",
    "K = settings['K']\n",
    "N_CLASSES = settings['n_classes']\n",
    "SEED = settings['seed']\n",
    "DROPOUT = settings['dropout']\n",
    "SPLIT = settings['split']\n",
    "SIZE = settings['size']\n",
    "PRETRAINED = settings['pretrained']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensorboard settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_run_log_dir(experiment_name): \n",
    "    log_dir = os.path.join(PATH, 'tb-runs') \n",
    "    if not os.path.exists(log_dir): \n",
    "        os.makedirs(log_dir) \n",
    "    run_log_dir = os.path.join(log_dir, experiment_name)\n",
    "    return run_log_dir\n",
    "\n",
    "log_dir = new_run_log_dir(EXPERIMENT_NAME)\n",
    "print(f'Tensorboard folder: {log_dir}')\n",
    "\n",
    "writer = SummaryWriter(log_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Handlers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clinical_data = f'{PATH}/data/clinical_data_noTx.csv'\n",
    "target_column = 'T-stage_binary'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(SEED)\n",
    "\n",
    "dataset = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, seed=SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create train-test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if SPLIT == 'valieres':\n",
    "    dataset_train = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode='train', transforms=augment_3D_HN)\n",
    "    \n",
    "    # in this particular case getting `dataset_train._files_full` or `dataset_train.get_files()` is the same\n",
    "    idx_train = [i for i, f in enumerate(dataset_train.get_files()) if f.split('-')[1] in ['CHUS', 'HGJ']]\n",
    "    dataset_train.indexes = np.array(idx_train)\n",
    "    \n",
    "    dataset_test = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode='test', transforms=augment_3D_HN)\n",
    "    \n",
    "    # in this particular case getting `dataset_train._files_full` or `dataset_train.get_files()` is the same\n",
    "    idx_test = [i for i, f in enumerate(dataset_test.get_files()) if f.split('-')[1] in ['HMR', 'CHUM']]\n",
    "    dataset_test.indexes = np.array(idx_test)\n",
    "                                      \n",
    "\n",
    "else:\n",
    "    idx_train, idx_test = train_test_indexes_patient_wise(dataset, test_size=K, stratify=True)\n",
    "    \n",
    "    dataset_test = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode='test', transforms=augment_3D_HN)\n",
    "    dataset_test.indexes = np.array(idx_test)\n",
    "\n",
    "    dataset_train = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode='train', transforms=augment_3D_HN)\n",
    "    dataset_train.indexes = np.array(idx_train)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check class balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_test = dataset_test.get_labels()\n",
    "labels_train = dataset_train.get_labels()\n",
    "\n",
    "c,n = np.unique(labels_test, return_counts=True)\n",
    "print(np.c_[c,n/len(labels_test)])\n",
    "\n",
    "c,n = np.unique(labels_train, return_counts=True)\n",
    "print(np.c_[c,n/len(labels_train)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create loaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE//2, num_workers=12, shuffle=True)\n",
    "loader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=12, pin_memory=True, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = dataset_train.get_labels()\n",
    "\n",
    "#class_sample_count = np.array([len(np.where( labels == t )[0]) for t in np.unique( labels )])\n",
    "_, class_sample_count = np.unique(labels, return_counts=True)\n",
    "n_min = np.min(class_sample_count)\n",
    "weights = n_min / class_sample_count # versione proporzionale, usare n_min invece che 1 per pesi ~1\n",
    "weights = torch.Tensor(weights).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MODEL(n_classes=N_CLASSES, n_channels=1, modality='CT', dropout=DROPOUT)\n",
    "\n",
    "if multigpu:\n",
    "    model = nn.DataParallel(model.to(device))\n",
    "    model = model.module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.initialize_weights()\n",
    "\n",
    "if PRETRAINED == 'Med3D':\n",
    "    pretrained_dict = torch.load(PRETRAINED_MED3D_WEIGHTS)['state_dict']\n",
    "    model_dict = model.state_dict()\n",
    "\n",
    "    # discard layers not present in destination network or with different shape\n",
    "    pretrained_dict = {k: v for k, v in pretrained_dict.items() if\n",
    "                           (k in model_dict) and (model_dict[k].shape == pretrained_dict[k].shape)}\n",
    "\n",
    "    for name in model.state_dict().keys():\n",
    "        if name in pretrained_dict.keys():\n",
    "            #print(name)\n",
    "            model.state_dict()[name].copy_(pretrained_dict[name])\n",
    "        \n",
    "elif PRETRAINED == 'branch-wise':\n",
    "    pretrained_CT_dict = torch.load(f'{EXPERIMENT_DIR}/Tstage_grouped_noTx_CT_20191021-143133/weights.pth')\n",
    "    pretrained_PT_dict = torch.load(f'{EXPERIMENT_DIR}/Tstage_binary_PET_noTx_20191022-124046/weights.pth')\n",
    "    \n",
    "    model_dict = model.state_dict()\n",
    "    \n",
    "    pretrained_CT_dict = {k: v for k, v in pretrained_CT_dict.items() if\n",
    "                           (k in model_dict) and (model_dict[k].shape == pretrained_CT_dict[k].shape)}\n",
    "    \n",
    "    pretrained_PT_dict = {k: v for k, v in pretrained_PT_dict.items() if\n",
    "                           (k in model_dict) and (model_dict[k].shape == pretrained_PT_dict[k].shape)}\n",
    "    \n",
    "    to_add = 'module.' if multigpu else ''\n",
    "    \n",
    "    for name in model.CT_branch.state_dict().keys():\n",
    "        name_complete = to_add + 'CT_branch.' + name\n",
    "        #print(name_complete)\n",
    "        if name_complete in pretrained_CT_dict.keys():\n",
    "            print(name)\n",
    "            model.CT_branch.state_dict()[name].copy_(pretrained_CT_dict[name_complete])\n",
    "            \n",
    "    for name in model.PT_branch.state_dict().keys():\n",
    "        name_complete = to_add + 'PT_branch.' + name\n",
    "        #print(name_complete)\n",
    "        if name_complete in pretrained_PT_dict.keys():\n",
    "            print(name)\n",
    "            model.PT_branch.state_dict()[name].copy_(pretrained_PT_dict[name_complete])\n",
    "\n",
    "elif PRETRAINED == 'T-stage':\n",
    "    pretrained_dict = torch.load(PRETRAINED_T_STAGE) \n",
    "    model_dict = model.state_dict()\n",
    "\n",
    "    # discard layers not present in destination network or with different shape\n",
    "    pretrained_dict = {k: v for k, v in pretrained_dict.items() if\n",
    "                           (k in model_dict) and (model_dict[k].shape == pretrained_dict[k].shape)}\n",
    "\n",
    "    for name in model.state_dict().keys():\n",
    "        if name in pretrained_dict.keys():\n",
    "            #print(name)\n",
    "            model.state_dict()[name].copy_(pretrained_dict[name])\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(), lr=LR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#[x.shape for x in model.parameters()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss(weight=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NEW_LABELS = list(range(len(list(np.unique(labels_train)))))\n",
    "dictionary = dict(zip(list(np.unique(labels_train)), NEW_LABELS))\n",
    "dictionary "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.train()  # Set model to training mode\n",
    "\n",
    "global_i = 0\n",
    "\n",
    "losses_tr = []\n",
    "losses_ts = []\n",
    "\n",
    "last_loss_test = -1\n",
    "iteration = 0\n",
    "start_time = time.time()\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "        #print(epoch)\n",
    "        if epoch % 10 == 0: #save checkpoint\n",
    "            torch.save(model.state_dict(), f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/checkpoint_{epoch}.pth')\n",
    "\n",
    "        for j, data in enumerate(loader_train):\n",
    "            global_i += 1\n",
    "            \n",
    "            if j%10 == 0:\n",
    "                print(time.time() - start_time)\n",
    "                start_time = time.time()\n",
    "\n",
    "        \n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            images_tr = data['data'].to(device)\n",
    "            labels_tr = torch.LongTensor([dictionary[i] for i in data['target']]).to(device)\n",
    "            outputs_tr = model(images_tr).to(device)\n",
    "\n",
    "            # backward\n",
    "            loss = criterion(outputs_tr, labels_tr)\n",
    "            loss.backward()\n",
    "\n",
    "            optimizer.step()\n",
    "\n",
    "            # check test set \n",
    "            if j % int(len(loader_train) / 2) == 0 and j != 0:\n",
    "                model.eval()\n",
    "                with torch.no_grad():\n",
    "\n",
    "                    losses_sum = 0\n",
    "                    num_samples_test = 0\n",
    "\n",
    "                    for data_test in loader_test:\n",
    "\n",
    "                        images_ts = data_test['data'].to(device)\n",
    "                        labels_ts = torch.LongTensor([dictionary[i] for i in data_test['target']]).to(device)\n",
    "\n",
    "                        outputs_ts = model.forward(images_ts)\n",
    "\n",
    "                        loss_test_sum = criterion(outputs_ts, labels_ts).item()\n",
    "                        losses_sum += loss_test_sum\n",
    "                        num_samples_test += 1\n",
    "\n",
    "                    loss_test_avg = losses_sum / num_samples_test\n",
    "                    \n",
    "                    writer.add_scalar(f'{EXPERIMENT_NAME}/test_loss', loss_test_avg, global_i)\n",
    "                    writer.flush()\n",
    "                    \n",
    "                    #is_best = loss_val_avg < last_loss_val\n",
    "                    #if is_best:\n",
    "                    #    torch.save(model.state_dict(),\n",
    "                    #               f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/checkpoint_best_{epoch}.pth')\n",
    "\n",
    "                    last_loss_test = loss_test_avg\n",
    "\n",
    "                losses_tr.append(loss.item())\n",
    "                losses_ts.append(loss_test_avg)\n",
    "\n",
    "                del images_ts, labels_ts\n",
    "\n",
    "            iteration += 1\n",
    "            del images_tr, labels_tr\n",
    "            gc.collect()\n",
    "            model.train()\n",
    "\n",
    "            # sys.stdout.write\n",
    "            writer.add_scalar(f'{EXPERIMENT_NAME}/train_loss', loss.item(), global_i)\n",
    "            writer.flush()\n",
    "            sys.stdout.write('\\r Epoch {} of {}  [{:.2f}%] - loss TR/TS: {:.4f} / {:.4f} - {}'.format(epoch + 1, EPOCHS,\n",
    "                                                                                                      100 * j / len(\n",
    "                                                                                                          loader_train),\n",
    "                                                                                                      loss.item(),\n",
    "                                                                                                      last_loss_test,\n",
    "                                                                                                      optimizer.param_groups[\n",
    "                                                                                                          0]['lr']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict on Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "dataset_train.mode = 'test' #no augmentation\n",
    "\n",
    "preds_tr = []\n",
    "trues_tr = []\n",
    "probs_tr = []\n",
    "filenames_tr = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for data in dataset_train:\n",
    "        image = data[\"data\"].unsqueeze(0).to(device)\n",
    "        label = data[\"target\"]\n",
    "        output = model(image) #forward\n",
    "        _, pred = torch.max(output,1)\n",
    "        \n",
    "        preds_tr.append(pred.data.cpu().numpy())\n",
    "#         trues.append(label)\n",
    "        trues_tr.append(dictionary[label])\n",
    "        probs_tr.append(output.data.cpu().numpy())\n",
    "        filenames_tr.append(data['filename'])\n",
    "\n",
    "probs_tr = np.concatenate(probs_tr)\n",
    "preds_tr = np.concatenate(preds_tr)\n",
    "trues_tr = np.array(trues_tr)\n",
    "filenames_tr = np.array(filenames_tr)\n",
    "\n",
    "MCC_tr = mcor(trues_tr, preds_tr)\n",
    "ACC_tr = acc(trues_tr, preds_tr)\n",
    "prec_tr = precision(trues_tr, preds_tr, average='weighted')\n",
    "rec_tr = recall(trues_tr, preds_tr, average='weighted')\n",
    "\n",
    "print(\"MCC train\", round(MCC_tr,3), \"ACC train\", round(ACC_tr, 3))\n",
    "print(\"precision train\", round(prec_tr, 3), \"recall train\", round(rec_tr, 3))\n",
    "\n",
    "train_metrics = [round(MCC_tr ,3), round(ACC_tr,3), round(prec_tr, 3), round(rec_tr, 3)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict on Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "\n",
    "preds_ts = []\n",
    "trues_ts = []\n",
    "probs_ts = []\n",
    "filenames_ts = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for data in dataset_test:\n",
    "        image = data[\"data\"].unsqueeze(0).to(device)\n",
    "        label = data[\"target\"]\n",
    "        output = model(image) #forward\n",
    "        _, pred = torch.max(output,1)\n",
    "        \n",
    "        preds_ts.append(pred.data.cpu().numpy())\n",
    "        trues_ts.append(dictionary[label])\n",
    "        probs_ts.append(output.data.cpu().numpy())\n",
    "        filenames_ts.append(data['filename'])\n",
    "\n",
    "probs_ts = np.concatenate(probs_ts)\n",
    "preds_ts = np.concatenate(preds_ts)\n",
    "trues_ts = np.array(trues_ts)\n",
    "filenames_ts = np.array(filenames_ts)\n",
    "\n",
    "MCC_ts = mcor(trues_ts, preds_ts)\n",
    "ACC_ts = acc(trues_ts, preds_ts)\n",
    "prec_ts = precision(trues_ts, preds_ts, average='weighted')\n",
    "rec_ts = recall(trues_ts, preds_ts, average='weighted')\n",
    "\n",
    "print(\"MCC train\", round(MCC_ts,3), \"ACC train\", round(ACC_ts, 3))\n",
    "print(\"precision train\", round(prec_ts, 3), \"recall train\", round(rec_ts, 3))\n",
    "train_metrics = [round(MCC_ts ,3), round(ACC_ts,3), round(prec_ts, 3), round(rec_ts, 3)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save settings\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/settings.pkl', 'wb') as f:\n",
    "    pickle.dump(settings, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save losses\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses_tr = np.array(losses_tr)\n",
    "losses_vl = np.array(losses_ts)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/losses_tr.npy', losses_tr)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/losses_ts.npy', losses_vl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "plt.plot(losses_tr, color='blue')\n",
    "plt.plot(losses_ts, color='orange')\n",
    "plt.legend(['train', 'valid'])\n",
    "plt.savefig(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/losses.png', close=True, verbose=True)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save predictions, ground truth, probabilities and filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/preds_tr.npy', preds_tr)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/trues_tr.npy', trues_tr)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/probs_tr.npy', probs_tr)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/filenames_tr.npy', filenames_tr)\n",
    "\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/preds_ts.npy', preds_ts)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/trues_ts.npy', trues_ts)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/probs_ts.npy', probs_ts)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/filenames_ts.npy', filenames_ts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "metrics_out = pd.DataFrame((train_metrics, test_metrics), columns=['MCC', 'ACC', 'prec', 'rec'], index = ['train','test'])\n",
    "metrics_out.to_csv(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/metrics_out.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save model weights"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (dappertf)",
   "language": "python",
   "name": "dappertf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}