Commit 7e68928f authored by Alessia Marcolini's avatar Alessia Marcolini
Browse files

PEP8 + use tqdm

parent 87f2a5d7
......@@ -9,30 +9,16 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<text style=\"color: green;\">Success</text>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"%HN_env"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
......@@ -42,7 +28,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
......@@ -59,84 +45,76 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import sys\n",
"import torch\n",
"from tqdm import tqdm\n",
"\n",
"import numpy as np\n",
"import os\n",
"import pandas as pd\n",
"import torch\n",
"import torch.nn as nn\n",
"from torch.utils.data import DataLoader\n",
"\n",
"from networks import CiompiDO\n",
"from dataset import NumpyCSVDataset, augment_3D_HN "
]
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
"os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
"\n",
"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
"multigpu = True"
]
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"DATASETDIR = \"/thunderdisk/HN/processed/bbox_fixed2_64\" #Not augmented but already 64**3 (for faster loading)\n",
"EXPERIMENT_DIR = f\"{PATH}/experiments\""
"DATASETDIR = '/thunderdisk/HN/processed/bbox_64_augmented_LR'\n",
"EXPERIMENT_DIR = f'{PATH}/experiments'"
]
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"MODEL_NAME = 'Tstage_binary_augmented_noTx_branch_wise_20191022-233016'\n",
"MODEL_NAME = 'LR_noTx_branch_wise_free_aug_20191027-003918'\n",
"SIZE = 64\n",
"\n",
"OUTDIR = f'{EXPERIMENT_DIR}/{MODEL_NAME}/features'\n",
"OUTFILE = 'features_wTx.csv'\n",
"# os.makedirs(OUTDIR, exist_ok=False)"
"OUTDIR = f'{EXPERIMENT_DIR}/{MODEL_NAME}/features/'\n",
"OUTFILE = 'features_noTx.csv'\n",
"os.makedirs(OUTDIR, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dataset = NumpyCSVDataset(DATASETDIR , f\"{PATH}/data/clinical_data.csv\" , \"T-stage_grouped\", SIZE , mode=\"test\")\n",
"dataset = NumpyCSVDataset(DATASETDIR , f'{PATH}/data/clinical_data_noTx.csv' , 'Locoregional', SIZE , mode='test')\n",
"loader = DataLoader(dataset, batch_size=16, num_workers=12, pin_memory=True, shuffle=False, drop_last=False)\n",
"model_weights = f'{EXPERIMENT_DIR}/{MODEL_NAME}/weights.pth'"
]
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<All keys matched successfully>"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"model = CiompiDO(n_classes=2, n_channels=2, modality='CT/PET')\n",
"\n",
......@@ -149,54 +127,9 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0\n",
"0\n",
"1\n",
"1\n",
"2\n",
"2\n",
"3\n",
"3\n",
"4\n",
"4\n",
"5\n",
"5\n",
"6\n",
"6\n",
"7\n",
"7\n",
"8\n",
"8\n",
"9\n",
"9\n",
"10\n",
"10\n",
"11\n",
"11\n",
"12\n",
"12\n",
"13\n",
"13\n",
"14\n",
"14\n",
"15\n",
"15\n",
"16\n",
"16\n",
"17\n",
"17\n",
"18\n",
"18\n"
]
}
],
"outputs": [],
"source": [
"#%%\n",
"deep_features = []\n",
......@@ -204,13 +137,11 @@
"labels = []\n",
"\n",
"with torch.no_grad():\n",
" for i,batch in enumerate(loader):\n",
" print(i)\n",
" names = batch['filename']\n",
" for batch in tqdm(loader):\n",
" names = [name.split('.')[0] for name in batch['filename']]\n",
" image = batch['data'].to(device)\n",
" label = batch['target']\n",
" out = model.extract_features(image.cuda())\n",
" print(i)\n",
" deep_features.append(out.data.cpu().numpy())\n",
" sample_names.append(names)\n",
" labels.append(label)"
......@@ -218,7 +149,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
......@@ -229,61 +160,41 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": null,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"294"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"outputs": [],
"source": [
"len(labels)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(294, 512) 294 294\n"
]
}
],
"outputs": [],
"source": [
"print(deep_features.shape, len(sample_names),len(labels))"
]
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(294, 513)\n"
]
}
],
"outputs": [],
"source": [
"deep_features_pd = pd.DataFrame(deep_features, index = sample_names)\n",
"deep_features_pd['class'] = labels \n",
"deep_features_pd = pd.DataFrame(deep_features, index=sample_names)\n",
"# deep_features_pd['class'] = labels \n",
"#%% SAVE\n",
"print(deep_features_pd.shape)\n",
"deep_features_pd.to_csv(f'{OUTDIR}/{OUTFILE}')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
......
%% Cell type:markdown id: tags:
## Deep features extraction
%% Cell type:code id: tags:
``` python
%HN_env
```
%%%% Output: execute_result
<IPython.core.display.HTML object>
%% Cell type:code id: tags:
``` python
import os
PATH = os.path.abspath(os.path.curdir)
```
%% Cell type:code id: tags:
``` python
%reload_ext autoreload
%autoreload 2
```
%% Cell type:markdown id: tags:
### Import
%% Cell type:code id: tags:
``` python
import os
import sys
import torch
from tqdm import tqdm
import numpy as np
import os
import pandas as pd
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from networks import CiompiDO
from dataset import NumpyCSVDataset, augment_3D_HN
```
%% Cell type:code id: tags:
``` python
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
multigpu = True
```
%% Cell type:code id: tags:
``` python
DATASETDIR = "/thunderdisk/HN/processed/bbox_fixed2_64" #Not augmented but already 64**3 (for faster loading)
EXPERIMENT_DIR = f"{PATH}/experiments"
DATASETDIR = '/thunderdisk/HN/processed/bbox_64_augmented_LR'
EXPERIMENT_DIR = f'{PATH}/experiments'
```
%% Cell type:code id: tags:
``` python
MODEL_NAME = 'Tstage_binary_augmented_noTx_branch_wise_20191022-233016'
MODEL_NAME = 'LR_noTx_branch_wise_free_aug_20191027-003918'
SIZE = 64
OUTDIR = f'{EXPERIMENT_DIR}/{MODEL_NAME}/features'
OUTFILE = 'features_wTx.csv'
# os.makedirs(OUTDIR, exist_ok=False)
OUTDIR = f'{EXPERIMENT_DIR}/{MODEL_NAME}/features/'
OUTFILE = 'features_noTx.csv'
os.makedirs(OUTDIR, exist_ok=True)
```
%% Cell type:code id: tags:
``` python
dataset = NumpyCSVDataset(DATASETDIR , f"{PATH}/data/clinical_data.csv" , "T-stage_grouped", SIZE , mode="test")
dataset = NumpyCSVDataset(DATASETDIR , f'{PATH}/data/clinical_data_noTx.csv' , 'Locoregional', SIZE , mode='test')
loader = DataLoader(dataset, batch_size=16, num_workers=12, pin_memory=True, shuffle=False, drop_last=False)
model_weights = f'{EXPERIMENT_DIR}/{MODEL_NAME}/weights.pth'
```
%% Cell type:code id: tags:
``` python
model = CiompiDO(n_classes=2, n_channels=2, modality='CT/PET')
if multigpu:
model = nn.DataParallel(model.to(device))
model = model.module
model.load_state_dict(torch.load(model_weights))
```
%%%% Output: execute_result
<All keys matched successfully>
%% Cell type:code id: tags:
``` python
#%%
deep_features = []
sample_names = []
labels = []
with torch.no_grad():
for i,batch in enumerate(loader):
print(i)
names = batch['filename']
for batch in tqdm(loader):
names = [name.split('.')[0] for name in batch['filename']]
image = batch['data'].to(device)
label = batch['target']
out = model.extract_features(image.cuda())
print(i)
deep_features.append(out.data.cpu().numpy())
sample_names.append(names)
labels.append(label)
```
%%%% Output: stream
0
0
1
1
2
2
3
3
4
4
5
5
6
6
7
7
8
8
9
9
10
10
11
11
12
12
13
13
14
14
15
15
16
16
17
17
18
18
%% Cell type:code id: tags:
``` python
deep_features = np.concatenate(deep_features)
sample_names = np.concatenate(sample_names)
labels = np.concatenate(labels)
```
%% Cell type:code id: tags:
``` python
len(labels)
```
%%%% Output: execute_result
294
%% Cell type:code id: tags:
``` python
print(deep_features.shape, len(sample_names),len(labels))
```
%%%% Output: stream
(294, 512) 294 294
%% Cell type:code id: tags:
``` python
deep_features_pd = pd.DataFrame(deep_features, index = sample_names)
deep_features_pd['class'] = labels
deep_features_pd = pd.DataFrame(deep_features, index=sample_names)
# deep_features_pd['class'] = labels
#%% SAVE
print(deep_features_pd.shape)
deep_features_pd.to_csv(f'{OUTDIR}/{OUTFILE}')
```
%%%% Output: stream
%% Cell type:code id: tags:
(294, 513)
``` python
```
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment