trainingTstage.ipynb 23.3 KB
Newer Older
Andrea Bizzego's avatar
Andrea Bizzego committed
1
2
3
4
5
6
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
7
    "## Training network for featture extraction"
Andrea Bizzego's avatar
Andrea Bizzego committed
8
9
10
11
12
13
14
15
16
17
18
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Path"
   ]
  },
  {
   "cell_type": "code",
19
20
21
   "execution_count": null,
   "metadata": {},
   "outputs": [],
Andrea Bizzego's avatar
Andrea Bizzego committed
22
23
24
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
25
    "\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
26
27
28
29
30
31
32
33
34
35
36
37
38
39
    "import os\n",
    "PATH = os.getcwd()\n",
    "print(PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import packages"
   ]
  },
  {
   "cell_type": "code",
40
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
41
42
43
   "metadata": {},
   "outputs": [],
   "source": [
44
45
46
    "import datetime\n",
    "import gc\n",
    "import pickle\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
47
    "import sys\n",
48
49
50
    "import time\n",
    "\n",
    "import numpy as np\n",
Alessia Marcolini's avatar
Alessia Marcolini committed
51
    "import matplotlib.pyplot as plt\n",
52
    "from sklearn.metrics import matthews_corrcoef as mcor, accuracy_score as acc, recall_score as recall, precision_score as precision, confusion_matrix\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
53
54
    "import torch\n",
    "import torch.nn as nn\n",
55
56
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
57
    "\n",
Alessia Marcolini's avatar
Alessia Marcolini committed
58
    "from networks import CiompiDO, ResNet50_3d\n",
59
60
    "from dataset import NumpyCSVDataset, augment_3D_HN \n",
    "from split import train_test_indexes_patient_wise"
Andrea Bizzego's avatar
Andrea Bizzego committed
61
62
63
64
   ]
  },
  {
   "cell_type": "code",
65
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
66
67
68
69
   "metadata": {},
   "outputs": [],
   "source": [
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1\"\n",
70
71
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "multigpu = True"
Andrea Bizzego's avatar
Andrea Bizzego committed
72
73
74
75
   ]
  },
  {
   "cell_type": "code",
76
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
77
78
79
   "metadata": {},
   "outputs": [],
   "source": [
80
    "DATASET_DIR = f\"/thunderdisk/HN/processed/bbox_fixed2_64/\" #Not augmented but already 64**3 (for faster loading)\n",
Alessia Marcolini's avatar
Alessia Marcolini committed
81
82
    "EXPERIMENT_DIR = f\"{PATH}/experiments\"\n",
    "\n",
83
84
    "PRETRAINED_MED3D_WEIGHTS = '/thunderdisk/HN/MedicalNet_pytorch_files/pretrain/resnet_50.pth'\n",
    "PRETRAINED_T_STAGE = f'{EXPERIMENT_DIR}/Tstage_binary_augmented_noTx_branch_wise_20191028-104101/checkpoint_40.pth'"
Andrea Bizzego's avatar
Andrea Bizzego committed
85
86
87
88
89
90
91
92
93
94
95
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Settings"
   ]
  },
  {
   "cell_type": "code",
96
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
97
98
99
   "metadata": {},
   "outputs": [],
   "source": [
100
    "EXPERIMENT_NAME = 'Tstage_grouped_noTx_CT_valieres_' + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
101
102
    "\n",
    "settings = {\n",
103
    "            'model': CiompiDO,\n",
104
105
    "            'batch_size': 32,\n",
    "            'lr': 1e-5,\n",
106
    "            'epochs': 300,\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
107
    "            'optim': torch.optim.Adam,\n",
108
    "            'K': 0.2,\n",
109
    "            'n_classes': 2, #TSTAGE\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
110
    "            'seed': 1234,\n",
111
    "            'dropout': 0.5,\n",
112
    "            'split': 'valieres',\n",
Alessia Marcolini's avatar
Alessia Marcolini committed
113
    "            'size': 64,\n",
114
    "            'pretrained': '',\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
115
116
    "            }\n",
    "\n",
117
118
119
    "assert settings['split'] in ['valeries', '8020']\n",
    "assert settings['pretrained'] in ['Med3D', 'branch-wise', 'T-stage', '']\n",
    "\n",
120
    "os.makedirs(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}', exist_ok=False)"
Andrea Bizzego's avatar
Andrea Bizzego committed
121
122
123
124
   ]
  },
  {
   "cell_type": "code",
125
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
126
127
128
129
130
131
132
133
134
135
136
137
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL = settings['model']\n",
    "BATCH_SIZE = settings['batch_size']\n",
    "LR = settings['lr']\n",
    "EPOCHS = settings['epochs']\n",
    "OPTIMIZER = settings['optim']\n",
    "K = settings['K']\n",
    "N_CLASSES = settings['n_classes']\n",
    "SEED = settings['seed']\n",
    "DROPOUT = settings['dropout']\n",
138
    "SPLIT = settings['split']\n",
Alessia Marcolini's avatar
Alessia Marcolini committed
139
140
    "SIZE = settings['size']\n",
    "PRETRAINED = settings['pretrained']"
Andrea Bizzego's avatar
Andrea Bizzego committed
141
142
   ]
  },
143
144
145
146
147
148
149
150
151
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensorboard settings"
   ]
  },
  {
   "cell_type": "code",
152
   "execution_count": null,
153
   "metadata": {},
154
   "outputs": [],
155
156
157
158
159
160
161
162
163
164
165
166
167
168
   "source": [
    "def new_run_log_dir(experiment_name): \n",
    "    log_dir = os.path.join(PATH, 'tb-runs') \n",
    "    if not os.path.exists(log_dir): \n",
    "        os.makedirs(log_dir) \n",
    "    run_log_dir = os.path.join(log_dir, experiment_name)\n",
    "    return run_log_dir\n",
    "\n",
    "log_dir = new_run_log_dir(EXPERIMENT_NAME)\n",
    "print(f'Tensorboard folder: {log_dir}')\n",
    "\n",
    "writer = SummaryWriter(log_dir)"
   ]
  },
Andrea Bizzego's avatar
Andrea Bizzego committed
169
170
171
172
173
174
175
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Handlers"
   ]
  },
176
177
  {
   "cell_type": "code",
178
   "execution_count": null,
179
180
181
   "metadata": {},
   "outputs": [],
   "source": [
Alessia Marcolini's avatar
Alessia Marcolini committed
182
183
    "clinical_data = f'{PATH}/data/clinical_data_noTx.csv'\n",
    "target_column = 'T-stage_binary'"
184
185
   ]
  },
Andrea Bizzego's avatar
Andrea Bizzego committed
186
187
  {
   "cell_type": "code",
188
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
189
190
191
192
193
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(SEED)\n",
    "\n",
194
    "dataset = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, seed=SEED)"
Andrea Bizzego's avatar
Andrea Bizzego committed
195
196
197
198
199
200
201
202
203
204
205
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create train-test datasets"
   ]
  },
  {
   "cell_type": "code",
206
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
207
208
209
   "metadata": {},
   "outputs": [],
   "source": [
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
    "if SPLIT == 'valieres':\n",
    "    dataset_train = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode='train', transforms=augment_3D_HN)\n",
    "    \n",
    "    # in this particular case getting `dataset_train._files_full` or `dataset_train.get_files()` is the same\n",
    "    idx_train = [i for i, f in enumerate(dataset_train.get_files()) if f.split('-')[1] in ['CHUS', 'HGJ']]\n",
    "    dataset_train.indexes = np.array(idx_train)\n",
    "    \n",
    "    dataset_test = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode='test', transforms=augment_3D_HN)\n",
    "    \n",
    "    # in this particular case getting `dataset_train._files_full` or `dataset_train.get_files()` is the same\n",
    "    idx_test = [i for i, f in enumerate(dataset_test.get_files()) if f.split('-')[1] in ['HMR', 'CHUM']]\n",
    "    dataset_test.indexes = np.array(idx_test)\n",
    "                                      \n",
    "\n",
    "else:\n",
    "    idx_train, idx_test = train_test_indexes_patient_wise(dataset, test_size=K, stratify=True)\n",
    "    \n",
    "    dataset_test = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode='test', transforms=augment_3D_HN)\n",
    "    dataset_test.indexes = np.array(idx_test)\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
229
    "\n",
230
231
    "    dataset_train = NumpyCSVDataset(DATASET_DIR, clinical_data, target_column, SIZE, mode='train', transforms=augment_3D_HN)\n",
    "    dataset_train.indexes = np.array(idx_train)\n"
Andrea Bizzego's avatar
Andrea Bizzego committed
232
233
234
235
236
237
238
239
240
241
242
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check class balance"
   ]
  },
  {
   "cell_type": "code",
243
244
245
   "execution_count": null,
   "metadata": {},
   "outputs": [],
Andrea Bizzego's avatar
Andrea Bizzego committed
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
   "source": [
    "labels_test = dataset_test.get_labels()\n",
    "labels_train = dataset_train.get_labels()\n",
    "\n",
    "c,n = np.unique(labels_test, return_counts=True)\n",
    "print(np.c_[c,n/len(labels_test)])\n",
    "\n",
    "c,n = np.unique(labels_train, return_counts=True)\n",
    "print(np.c_[c,n/len(labels_train)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create loaders"
   ]
  },
  {
   "cell_type": "code",
266
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
267
268
269
   "metadata": {},
   "outputs": [],
   "source": [
270
    "loader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE//2, num_workers=12, shuffle=True)\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
271
272
273
274
275
276
277
278
279
280
281
282
    "loader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, num_workers=12, pin_memory=True, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute weights"
   ]
  },
  {
   "cell_type": "code",
283
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = dataset_train.get_labels()\n",
    "\n",
    "#class_sample_count = np.array([len(np.where( labels == t )[0]) for t in np.unique( labels )])\n",
    "_, class_sample_count = np.unique(labels, return_counts=True)\n",
    "n_min = np.min(class_sample_count)\n",
    "weights = n_min / class_sample_count # versione proporzionale, usare n_min invece che 1 per pesi ~1\n",
    "weights = torch.Tensor(weights).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
300
    "### Initialize Model"
Andrea Bizzego's avatar
Andrea Bizzego committed
301
302
303
   ]
  },
  {
304
305
   "cell_type": "code",
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
306
   "metadata": {},
307
   "outputs": [],
Andrea Bizzego's avatar
Andrea Bizzego committed
308
   "source": [
309
    "model = MODEL(n_classes=N_CLASSES, n_channels=1, modality='CT', dropout=DROPOUT)\n",
310
311
312
313
    "\n",
    "if multigpu:\n",
    "    model = nn.DataParallel(model.to(device))\n",
    "    model = model.module"
Andrea Bizzego's avatar
Andrea Bizzego committed
314
315
316
317
   ]
  },
  {
   "cell_type": "code",
318
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
319
320
321
   "metadata": {},
   "outputs": [],
   "source": [
322
    "model.initialize_weights()\n",
Alessia Marcolini's avatar
Alessia Marcolini committed
323
    "\n",
324
    "if PRETRAINED == 'Med3D':\n",
Alessia Marcolini's avatar
Alessia Marcolini committed
325
326
327
328
329
330
    "    pretrained_dict = torch.load(PRETRAINED_MED3D_WEIGHTS)['state_dict']\n",
    "    model_dict = model.state_dict()\n",
    "\n",
    "    # discard layers not present in destination network or with different shape\n",
    "    pretrained_dict = {k: v for k, v in pretrained_dict.items() if\n",
    "                           (k in model_dict) and (model_dict[k].shape == pretrained_dict[k].shape)}\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
331
    "\n",
Alessia Marcolini's avatar
Alessia Marcolini committed
332
333
334
335
    "    for name in model.state_dict().keys():\n",
    "        if name in pretrained_dict.keys():\n",
    "            #print(name)\n",
    "            model.state_dict()[name].copy_(pretrained_dict[name])\n",
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
    "        \n",
    "elif PRETRAINED == 'branch-wise':\n",
    "    pretrained_CT_dict = torch.load(f'{EXPERIMENT_DIR}/Tstage_grouped_noTx_CT_20191021-143133/weights.pth')\n",
    "    pretrained_PT_dict = torch.load(f'{EXPERIMENT_DIR}/Tstage_binary_PET_noTx_20191022-124046/weights.pth')\n",
    "    \n",
    "    model_dict = model.state_dict()\n",
    "    \n",
    "    pretrained_CT_dict = {k: v for k, v in pretrained_CT_dict.items() if\n",
    "                           (k in model_dict) and (model_dict[k].shape == pretrained_CT_dict[k].shape)}\n",
    "    \n",
    "    pretrained_PT_dict = {k: v for k, v in pretrained_PT_dict.items() if\n",
    "                           (k in model_dict) and (model_dict[k].shape == pretrained_PT_dict[k].shape)}\n",
    "    \n",
    "    to_add = 'module.' if multigpu else ''\n",
    "    \n",
    "    for name in model.CT_branch.state_dict().keys():\n",
    "        name_complete = to_add + 'CT_branch.' + name\n",
    "        #print(name_complete)\n",
    "        if name_complete in pretrained_CT_dict.keys():\n",
    "            print(name)\n",
    "            model.CT_branch.state_dict()[name].copy_(pretrained_CT_dict[name_complete])\n",
    "            \n",
    "    for name in model.PT_branch.state_dict().keys():\n",
    "        name_complete = to_add + 'PT_branch.' + name\n",
    "        #print(name_complete)\n",
    "        if name_complete in pretrained_PT_dict.keys():\n",
    "            print(name)\n",
    "            model.PT_branch.state_dict()[name].copy_(pretrained_PT_dict[name_complete])\n",
364
365
366
367
368
369
370
371
372
373
374
375
376
    "\n",
    "elif PRETRAINED == 'T-stage':\n",
    "    pretrained_dict = torch.load(PRETRAINED_T_STAGE) \n",
    "    model_dict = model.state_dict()\n",
    "\n",
    "    # discard layers not present in destination network or with different shape\n",
    "    pretrained_dict = {k: v for k, v in pretrained_dict.items() if\n",
    "                           (k in model_dict) and (model_dict[k].shape == pretrained_dict[k].shape)}\n",
    "\n",
    "    for name in model.state_dict().keys():\n",
    "        if name in pretrained_dict.keys():\n",
    "            #print(name)\n",
    "            model.state_dict()[name].copy_(pretrained_dict[name])\n",
377
    "            "
Andrea Bizzego's avatar
Andrea Bizzego committed
378
379
380
381
382
383
384
385
386
387
388
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Optimizer"
   ]
  },
  {
   "cell_type": "code",
389
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(), lr=LR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#[x.shape for x in model.parameters()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loss"
   ]
  },
  {
   "cell_type": "code",
414
415
416
417
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
Alessia Marcolini's avatar
Alessia Marcolini committed
418
    "criterion = nn.CrossEntropyLoss(weight=weights)"
419
420
421
422
423
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
424
425
426
   "metadata": {},
   "outputs": [],
   "source": [
427
428
429
    "NEW_LABELS = list(range(len(list(np.unique(labels_train)))))\n",
    "dictionary = dict(zip(list(np.unique(labels_train)), NEW_LABELS))\n",
    "dictionary "
Andrea Bizzego's avatar
Andrea Bizzego committed
430
431
432
433
434
435
436
437
438
439
440
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
441
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
442
443
444
   "metadata": {},
   "outputs": [],
   "source": [
445
446
447
    "model.train()  # Set model to training mode\n",
    "\n",
    "global_i = 0\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
448
449
    "\n",
    "losses_tr = []\n",
450
451
452
    "losses_ts = []\n",
    "\n",
    "last_loss_test = -1\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
453
    "iteration = 0\n",
454
455
    "start_time = time.time()\n",
    "\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
456
    "for epoch in range(EPOCHS):\n",
457
458
459
460
461
462
463
464
465
466
467
    "        #print(epoch)\n",
    "        if epoch % 10 == 0: #save checkpoint\n",
    "            torch.save(model.state_dict(), f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/checkpoint_{epoch}.pth')\n",
    "\n",
    "        for j, data in enumerate(loader_train):\n",
    "            global_i += 1\n",
    "            \n",
    "            if j%10 == 0:\n",
    "                print(time.time() - start_time)\n",
    "                start_time = time.time()\n",
    "\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
468
    "        \n",
469
    "            optimizer.zero_grad()\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
470
    "\n",
471
472
473
    "            images_tr = data['data'].to(device)\n",
    "            labels_tr = torch.LongTensor([dictionary[i] for i in data['target']]).to(device)\n",
    "            outputs_tr = model(images_tr).to(device)\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
474
    "\n",
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
    "            # backward\n",
    "            loss = criterion(outputs_tr, labels_tr)\n",
    "            loss.backward()\n",
    "\n",
    "            optimizer.step()\n",
    "\n",
    "            # check test set \n",
    "            if j % int(len(loader_train) / 2) == 0 and j != 0:\n",
    "                model.eval()\n",
    "                with torch.no_grad():\n",
    "\n",
    "                    losses_sum = 0\n",
    "                    num_samples_test = 0\n",
    "\n",
    "                    for data_test in loader_test:\n",
    "\n",
    "                        images_ts = data_test['data'].to(device)\n",
    "                        labels_ts = torch.LongTensor([dictionary[i] for i in data_test['target']]).to(device)\n",
    "\n",
    "                        outputs_ts = model.forward(images_ts)\n",
    "\n",
    "                        loss_test_sum = criterion(outputs_ts, labels_ts).item()\n",
    "                        losses_sum += loss_test_sum\n",
    "                        num_samples_test += 1\n",
    "\n",
    "                    loss_test_avg = losses_sum / num_samples_test\n",
    "                    \n",
    "                    writer.add_scalar(f'{EXPERIMENT_NAME}/test_loss', loss_test_avg, global_i)\n",
    "                    writer.flush()\n",
    "                    \n",
    "                    #is_best = loss_val_avg < last_loss_val\n",
    "                    #if is_best:\n",
    "                    #    torch.save(model.state_dict(),\n",
    "                    #               f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/checkpoint_best_{epoch}.pth')\n",
    "\n",
    "                    last_loss_test = loss_test_avg\n",
    "\n",
    "                losses_tr.append(loss.item())\n",
    "                losses_ts.append(loss_test_avg)\n",
    "\n",
    "                del images_ts, labels_ts\n",
    "\n",
    "            iteration += 1\n",
    "            del images_tr, labels_tr\n",
    "            gc.collect()\n",
    "            model.train()\n",
    "\n",
    "            # sys.stdout.write\n",
    "            writer.add_scalar(f'{EXPERIMENT_NAME}/train_loss', loss.item(), global_i)\n",
    "            writer.flush()\n",
    "            sys.stdout.write('\\r Epoch {} of {}  [{:.2f}%] - loss TR/TS: {:.4f} / {:.4f} - {}'.format(epoch + 1, EPOCHS,\n",
    "                                                                                                      100 * j / len(\n",
    "                                                                                                          loader_train),\n",
    "                                                                                                      loss.item(),\n",
    "                                                                                                      last_loss_test,\n",
    "                                                                                                      optimizer.param_groups[\n",
    "                                                                                                          0]['lr']))"
Andrea Bizzego's avatar
Andrea Bizzego committed
532
533
534
535
536
537
538
539
540
541
542
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict on Train"
   ]
  },
  {
   "cell_type": "code",
543
544
545
   "execution_count": null,
   "metadata": {},
   "outputs": [],
Andrea Bizzego's avatar
Andrea Bizzego committed
546
547
548
   "source": [
    "model.eval()\n",
    "dataset_train.mode = 'test' #no augmentation\n",
549
550
551
552
553
554
    "\n",
    "preds_tr = []\n",
    "trues_tr = []\n",
    "probs_tr = []\n",
    "filenames_tr = []\n",
    "\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
555
    "with torch.no_grad():\n",
556
557
558
    "    for data in dataset_train:\n",
    "        image = data[\"data\"].unsqueeze(0).to(device)\n",
    "        label = data[\"target\"]\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
559
560
561
    "        output = model(image) #forward\n",
    "        _, pred = torch.max(output,1)\n",
    "        \n",
562
    "        preds_tr.append(pred.data.cpu().numpy())\n",
563
    "#         trues.append(label)\n",
564
565
    "        trues_tr.append(dictionary[label])\n",
    "        probs_tr.append(output.data.cpu().numpy())\n",
566
    "        filenames_tr.append(data['filename'])\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
567
    "\n",
568
569
570
571
    "probs_tr = np.concatenate(probs_tr)\n",
    "preds_tr = np.concatenate(preds_tr)\n",
    "trues_tr = np.array(trues_tr)\n",
    "filenames_tr = np.array(filenames_tr)\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
572
    "\n",
573
574
575
576
    "MCC_tr = mcor(trues_tr, preds_tr)\n",
    "ACC_tr = acc(trues_tr, preds_tr)\n",
    "prec_tr = precision(trues_tr, preds_tr, average='weighted')\n",
    "rec_tr = recall(trues_tr, preds_tr, average='weighted')\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
577
    "\n",
578
579
580
581
    "print(\"MCC train\", round(MCC_tr,3), \"ACC train\", round(ACC_tr, 3))\n",
    "print(\"precision train\", round(prec_tr, 3), \"recall train\", round(rec_tr, 3))\n",
    "\n",
    "train_metrics = [round(MCC_tr ,3), round(ACC_tr,3), round(prec_tr, 3), round(rec_tr, 3)]"
Andrea Bizzego's avatar
Andrea Bizzego committed
582
583
584
585
586
587
588
589
590
591
592
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict on Test"
   ]
  },
  {
   "cell_type": "code",
593
594
595
   "execution_count": null,
   "metadata": {},
   "outputs": [],
Andrea Bizzego's avatar
Andrea Bizzego committed
596
597
598
   "source": [
    "model.eval()\n",
    "\n",
599
600
601
602
603
    "preds_ts = []\n",
    "trues_ts = []\n",
    "probs_ts = []\n",
    "filenames_ts = []\n",
    "\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
604
    "with torch.no_grad():\n",
605
606
607
    "    for data in dataset_test:\n",
    "        image = data[\"data\"].unsqueeze(0).to(device)\n",
    "        label = data[\"target\"]\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
608
609
610
    "        output = model(image) #forward\n",
    "        _, pred = torch.max(output,1)\n",
    "        \n",
611
612
613
    "        preds_ts.append(pred.data.cpu().numpy())\n",
    "        trues_ts.append(dictionary[label])\n",
    "        probs_ts.append(output.data.cpu().numpy())\n",
614
    "        filenames_ts.append(data['filename'])\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
615
    "\n",
616
617
618
619
    "probs_ts = np.concatenate(probs_ts)\n",
    "preds_ts = np.concatenate(preds_ts)\n",
    "trues_ts = np.array(trues_ts)\n",
    "filenames_ts = np.array(filenames_ts)\n",
Andrea Bizzego's avatar
Andrea Bizzego committed
620
    "\n",
621
622
623
624
    "MCC_ts = mcor(trues_ts, preds_ts)\n",
    "ACC_ts = acc(trues_ts, preds_ts)\n",
    "prec_ts = precision(trues_ts, preds_ts, average='weighted')\n",
    "rec_ts = recall(trues_ts, preds_ts, average='weighted')\n",
625
626
627
628
    "\n",
    "print(\"MCC train\", round(MCC_ts,3), \"ACC train\", round(ACC_ts, 3))\n",
    "print(\"precision train\", round(prec_ts, 3), \"recall train\", round(rec_ts, 3))\n",
    "train_metrics = [round(MCC_ts ,3), round(ACC_ts,3), round(prec_ts, 3), round(rec_ts, 3)]"
Andrea Bizzego's avatar
Andrea Bizzego committed
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save settings\n"
   ]
  },
  {
   "cell_type": "code",
647
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/settings.pkl', 'wb') as f:\n",
    "    pickle.dump(settings, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save losses\n"
   ]
  },
  {
   "cell_type": "code",
664
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
   "metadata": {},
   "outputs": [],
   "source": [
    "losses_tr = np.array(losses_tr)\n",
    "losses_vl = np.array(losses_ts)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/losses_tr.npy', losses_tr)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/losses_ts.npy', losses_vl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot losses"
   ]
  },
  {
   "cell_type": "code",
683
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "plt.plot(losses_tr, color='blue')\n",
    "plt.plot(losses_ts, color='orange')\n",
    "plt.legend(['train', 'valid'])\n",
    "plt.savefig(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/losses.png', close=True, verbose=True)\n",
    "plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save predictions, ground truth, probabilities and filenames"
   ]
  },
  {
   "cell_type": "code",
704
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
705
706
707
708
709
710
711
712
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/preds_tr.npy', preds_tr)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/trues_tr.npy', trues_tr)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/probs_tr.npy', probs_tr)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/filenames_tr.npy', filenames_tr)\n",
    "\n",
713
714
715
716
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/preds_ts.npy', preds_ts)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/trues_ts.npy', trues_ts)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/probs_ts.npy', probs_ts)\n",
    "np.save(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/filenames_ts.npy', filenames_ts)"
Andrea Bizzego's avatar
Andrea Bizzego committed
717
718
719
720
721
722
723
724
725
726
727
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save metrics"
   ]
  },
  {
   "cell_type": "code",
728
   "execution_count": null,
Andrea Bizzego's avatar
Andrea Bizzego committed
729
730
731
   "metadata": {},
   "outputs": [],
   "source": [
732
733
734
    "import pandas as pd\n",
    "metrics_out = pd.DataFrame((train_metrics, test_metrics), columns=['MCC', 'ACC', 'prec', 'rec'], index = ['train','test'])\n",
    "metrics_out.to_csv(f'{EXPERIMENT_DIR}/{EXPERIMENT_NAME}/metrics_out.csv', index=False)"
Andrea Bizzego's avatar
Andrea Bizzego committed
735
736
737
738
739
740
741
742
743
744
745
746
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save model weights"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
747
   "display_name": "Python (dappertf)",
Andrea Bizzego's avatar
Andrea Bizzego committed
748
   "language": "python",
749
   "name": "dappertf"
Andrea Bizzego's avatar
Andrea Bizzego committed
750
751
752
753
754
755
756
757
758
759
760
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
761
   "version": "3.6.9"
Andrea Bizzego's avatar
Andrea Bizzego committed
762
763
764
765
766
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}