Commit bcd3b239 authored by Marco Cristoforetti's avatar Marco Cristoforetti
Browse files

new network

parent 801ae04a
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
%matplotlib inline %matplotlib inline
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import sys; sys.path.append('../DST') import sys; sys.path.append('../DST')
import os import os
from DST.config import data_path from DST.config import data_path
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import seaborn as sns; sns.set(style="whitegrid", font_scale=1.3) import seaborn as sns; sns.set(style="whitegrid", font_scale=1.3)
import torch import torch
import torch.nn as nn import torch.nn as nn
import time import time
import math import math
import torch.utils.data as utils_data import torch.utils.data as utils_data
import torch.nn.functional as F import torch.nn.functional as F
import datetime import datetime
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
torch.manual_seed(21894) torch.manual_seed(21894)
np.random.seed(21894) np.random.seed(21894)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
BEFORE = 12 BEFORE = 12
AFTER = 12 AFTER = 12
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dst_data = pd.read_pickle(os.path.join(data_path,'dst.pkl')) dst_data = pd.read_pickle(os.path.join(data_path,'dst.pkl'))
dst_data['ora_round'] = dst_data.ora.apply(lambda x:int(x.split(':')[0])) dst_data['ora_round'] = dst_data.ora.apply(lambda x:int(x.split(':')[0]))
dati_agg = dst_data.groupby(['data','ora_round']).agg({ dati_agg = dst_data.groupby(['data','ora_round']).agg({
'BX': np.mean, 'BX': np.mean,
'BY': np.mean, 'BY': np.mean,
'BZ': np.mean, 'BZ': np.mean,
'FLOW_SPEED': np.mean, 'FLOW_SPEED': np.mean,
'PROTON_DENSITY': np.mean, 'PROTON_DENSITY': np.mean,
'TEMPERATURE': np.mean, 'TEMPERATURE': np.mean,
'PRESSION': np.mean, 'PRESSION': np.mean,
'ELETTRIC': np.mean, 'ELETTRIC': np.mean,
'y': np.mean}) 'y': np.mean})
dati_agg.reset_index(inplace=True) dati_agg.reset_index(inplace=True)
dati_agg.sort_values(by = ['data','ora_round'],inplace=True) dati_agg.sort_values(by = ['data','ora_round'],inplace=True)
dataset = dati_agg.drop(columns = ['data','ora_round']).values dataset = dati_agg.drop(columns = ['data','ora_round']).values
dataset = torch.from_numpy(np.hstack([np.arange(len(dataset)).reshape([-1,1]),dataset])) dataset = torch.from_numpy(np.hstack([np.arange(len(dataset)).reshape([-1,1]),dataset]))
last_date_train = dati_agg[dati_agg.data <= datetime.datetime(2008,12,31)].index[-1] last_date_train = dati_agg[dati_agg.data <= datetime.datetime(2008,12,31)].index[-1]
len_valid_test = (len(dataset) - last_date_train)/2 len_valid_test = (len(dataset) - last_date_train)/2
last_date_train/len(dataset), len_valid_test/len(dataset) last_date_train/len(dataset), len_valid_test/len(dataset)
data_in = dataset.unfold(0, BEFORE, 1).transpose(2,1) data_in = dataset.unfold(0, BEFORE, 1).transpose(2,1)
data_out = dataset[BEFORE:].unfold(0, AFTER, 1).transpose(2,1) data_out = dataset[BEFORE:].unfold(0, AFTER, 1).transpose(2,1)
data_in = data_in[:data_out.size(0)] data_in = data_in[:data_out.size(0)]
data_out = data_out[:,:,-1] data_out = data_out[:,:,-1]
data_in.size(), data_out.size() data_in.size(), data_out.size()
``` ```
%% Output
(torch.Size([261794, 12, 10]), torch.Size([261794, 12]))
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
where_not_nan_in = ~torch.isnan(data_in).any(2, keepdim=True).any(1, keepdim=True).reshape(-1) where_not_nan_in = ~torch.isnan(data_in).any(2, keepdim=True).any(1, keepdim=True).reshape(-1)
data_in = data_in[where_not_nan_in] data_in = data_in[where_not_nan_in]
data_out = data_out[where_not_nan_in] data_out = data_out[where_not_nan_in]
where_not_nan_out = ~torch.isnan(data_out).any(1, keepdim=True).reshape(-1) where_not_nan_out = ~torch.isnan(data_out).any(1, keepdim=True).reshape(-1)
data_in = data_in[where_not_nan_out] data_in = data_in[where_not_nan_out]
data_out = data_out[where_not_nan_out] data_out = data_out[where_not_nan_out]
last_train = np.where(data_in[:,0,0] <= last_date_train)[0][-1] + 1 last_train = np.where(data_in[:,0,0] <= last_date_train)[0][-1] + 1
data_in = data_in[:, :, 1:] data_in = data_in[:, :, 1:]
#len_tr = int(data_in.size(0) * 0.6) #len_tr = int(data_in.size(0) * 0.6)
n_channels = data_in.size(2) n_channels = data_in.size(2)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
class MinMaxScaler(): class MinMaxScaler():
""" """
Transform features by scaling each feature to a given range Transform features by scaling each feature to a given range
Features in the last dim Features in the last dim
The transformation is given by:: The transformation is given by::
X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0)) X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
X_scaled = X_std * (max - min) + min X_scaled = X_std * (max - min) + min
where min, max = feature_range. where min, max = feature_range.
""" """
def __init__(self, feature_range=(0,1)): def __init__(self, feature_range=(0,1)):
self.feature_range = feature_range self.feature_range = feature_range
def fit(self, X): def fit(self, X):
X_size = X.size() X_size = X.size()
X = X.reshape(-1, X_size[-1]) X = X.reshape(-1, X_size[-1])
data_min = X.min(axis=0).values data_min = X.min(axis=0).values
data_max = X.max(axis=0).values data_max = X.max(axis=0).values
data_range = data_max - data_min data_range = data_max - data_min
self.scale_ = ((self.feature_range[1] - self.feature_range[0]) / data_range) self.scale_ = ((self.feature_range[1] - self.feature_range[0]) / data_range)
self.min_ = self.feature_range[0] - data_min * self.scale_ self.min_ = self.feature_range[0] - data_min * self.scale_
self.data_min_ = data_min self.data_min_ = data_min
self.data_max_ = data_max self.data_max_ = data_max
self.data_range_ = data_range self.data_range_ = data_range
X = X.reshape(X_size) X = X.reshape(X_size)
return self return self
def transform(self, X): def transform(self, X):
X *= self.scale_ X *= self.scale_
X += self.min_ X += self.min_
return X return X
def inverse_transform(self, X): def inverse_transform(self, X):
X -= self.min_ X -= self.min_
X /= self.scale_ X /= self.scale_
return X return X
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
mmScaler = MinMaxScaler((0.1, .9)) mmScaler = MinMaxScaler((0.1, .9))
mmScaler.fit(data_in[:last_train]) mmScaler.fit(data_in[:last_train])
data_in_scaled = data_in.clone() data_in_scaled = data_in.clone()
data_in_scaled = mmScaler.transform(data_in_scaled) data_in_scaled = mmScaler.transform(data_in_scaled)
mm_scaler_out = MinMaxScaler((0.1, .9)) mm_scaler_out = MinMaxScaler((0.1, .9))
mm_scaler_out.fit(data_in[:last_train, :, -1].reshape(-1, data_in.size(1), 1)) mm_scaler_out.fit(data_in[:last_train, :, -1].reshape(-1, data_in.size(1), 1))
data_out_scaled = data_out.clone() data_out_scaled = data_out.clone()
data_out_scaled = mm_scaler_out.transform(data_out_scaled) data_out_scaled = mm_scaler_out.transform(data_out_scaled)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dst_levels = [-20,-50,-100] dst_levels = [-20,-50,-100]
data_out_c = data_out.clone() data_out_c = data_out.clone()
data_out_c[np.where(data_out_c >= dst_levels[0])] = 0 data_out_c[np.where(data_out_c >= dst_levels[0])] = 0
data_out_c[np.where((data_out_c < dst_levels[0]) & (data_out_c >= dst_levels[1]))] = 1 data_out_c[np.where((data_out_c < dst_levels[0]) & (data_out_c >= dst_levels[1]))] = 1
data_out_c[np.where((data_out_c < dst_levels[1]) & (data_out_c >= dst_levels[2]))] = 2 data_out_c[np.where((data_out_c < dst_levels[1]) & (data_out_c >= dst_levels[2]))] = 2
data_out_c[np.where((data_out_c < dst_levels[2]))] = 3 data_out_c[np.where((data_out_c < dst_levels[2]))] = 3
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
class Dataset(utils_data.Dataset): class Dataset(utils_data.Dataset):
def __init__(self, dataset_in, dataset_out, dataset_out_c): def __init__(self, dataset_in, dataset_out, dataset_out_c, weights):
self.dataset_in = dataset_in self.dataset_in = dataset_in
self.dataset_out = dataset_out self.dataset_out = dataset_out
self.dataset_out_c = dataset_out_c self.dataset_out_c = dataset_out_c
self.weights = weights
def __len__(self): def __len__(self):
return self.dataset_in.size(0) return self.dataset_in.size(0)
def __getitem__(self, idx): def __getitem__(self, idx):
din_src = self.dataset_in[idx] din_src = self.dataset_in[idx]
dout = self.dataset_out[idx] dout = self.dataset_out[idx]
dout_c = self.dataset_out_c[idx] dout_c = self.dataset_out_c[idx]
return din_src, dout, dout_c ww = self.weights[idx]
return din_src, dout, dout_c, ww
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
ixs_valid_test = np.arange(int(len_valid_test)) + last_train ixs_valid_test = np.arange(int(len_valid_test)) + last_train
np.random.shuffle(ixs_valid_test) np.random.shuffle(ixs_valid_test)
ixs_valid = ixs_valid_test[::2] ixs_valid = ixs_valid_test[::2]
ixs_test = ixs_valid_test[1::2] ixs_test = ixs_valid_test[1::2]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dst_min = data_out[:last_train].min(axis=1).values.flatten() dst_min = data_out[:last_train].min(axis=1).values.flatten()
bins = [dst_min.min() - 10] + list(np.arange(-300, dst_min.max() + 10, 10)) bins = [dst_min.min() - 10] + list(np.arange(-300, dst_min.max() + 10, 10))
h, b = np.histogram(dst_min, bins=bins) h, b = np.histogram(dst_min, bins=bins)
if len(np.argwhere(h == 0)) > 0: if len(np.argwhere(h == 0)) > 0:
bins = np.delete(bins, np.argwhere(h == 0)[0] + 1) bins = np.delete(bins, np.argwhere(h == 0)[0] + 1)
h, b = np.histogram(dst_min, bins=bins) h, b = np.histogram(dst_min, bins=bins)
w = h.max()/h w = h.max()/h
def fix_weight(dst_v): def fix_weight(dst_v):
pos = np.argwhere(np.abs(b - dst_v) == np.abs((b - dst_v)).min())[0,0] pos = np.argwhere(np.abs(b - dst_v) == np.abs((b - dst_v)).min())[0,0]
if dst_v - b[pos] < 0: if dst_v - b[pos] < 0:
pos = pos-1 pos = pos-1
# return w[pos]/h.max() # return w[pos]/h.max()
return np.sqrt(w[pos]/h.max()) return np.sqrt(w[pos]/h.max())
fix_weight_v = np.vectorize(fix_weight) fix_weight_v = np.vectorize(fix_weight)
weights = fix_weight_v(dst_min) weights = fix_weight_v(dst_min)
sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, num_samples= len(dst_min)) sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, num_samples= len(dst_min))
BATCH_SIZE=256 BATCH_SIZE=256
dataset_tr = Dataset(data_in_scaled[:last_train], data_out_scaled[:last_train], data_out_c[:last_train]) dataset_tr = Dataset(data_in_scaled[:last_train], data_out_scaled[:last_train], data_out_c[:last_train], weights)
# data_loader_tr = utils_data.DataLoader(dataset_tr, batch_size=BATCH_SIZE, num_workers = 4, shuffle=False, sampler = sampler) # data_loader_tr = utils_data.DataLoader(dataset_tr, batch_size=BATCH_SIZE, num_workers = 4, shuffle=False, sampler = sampler)
data_loader_tr = utils_data.DataLoader(dataset_tr, batch_size=BATCH_SIZE, num_workers = 4, shuffle=True) data_loader_tr = utils_data.DataLoader(dataset_tr, batch_size=BATCH_SIZE, num_workers = 4, shuffle=True)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
aa = next(iter(data_loader_tr))#[0].size()
```
%% Cell type:code id: tags:
``` python
class DSTnet(nn.Module): class DSTnet(nn.Module):
def __init__(self, nvars, nhidden_i, nhidden_o, n_out_i, before, after): def __init__(self, nvars, nhidden_i, nhidden_o, n_out_i, before, after):
super().__init__() super().__init__()
self.nvars = nvars self.nvars = nvars
self.nhidden_i = nhidden_i self.nhidden_i = nhidden_i
self.nhidden_o = nhidden_o self.nhidden_o = nhidden_o
self.before = before self.before = before
self.after = after self.after = after
self.n_out_i = n_out_i self.n_out_i = n_out_i
self.lstm = nn.LSTM(self.nvars, self.n_out_i, self.nhidden_i, batch_first=True) self.lstm = nn.LSTM(self.nvars, self.n_out_i, self.nhidden_i, batch_first=True)
self.first_merged_layer = self.n_out_i * self.before self.first_merged_layer = self.n_out_i * self.before
self.bn1 = nn.BatchNorm1d(num_features=self.first_merged_layer) self.bn1 = nn.BatchNorm1d(num_features=self.first_merged_layer)
self.linear_o_1 = nn.Linear(self.first_merged_layer, self.nhidden_o) self.linear_o_1 = nn.Linear(self.first_merged_layer, self.nhidden_o)
self.ln1 = nn.LayerNorm(self.nhidden_o ) self.ln1 = nn.LayerNorm(self.nhidden_o )
self.linear_o_2 = nn.Linear(self.nhidden_o, self.nhidden_o) self.linear_o_2 = nn.Linear(self.nhidden_o, self.nhidden_o)
self.linear_o_3 = nn.Linear(self.nhidden_o, self.nhidden_o // 2) self.linear_o_3 = nn.Linear(self.nhidden_o, self.nhidden_o // 2)
self.linear_o_4 = nn.Linear(self.nhidden_o // 2, self.after) self.linear_o_4 = nn.Linear(self.nhidden_o // 2, self.after)
self.linear_o_3_c = nn.Linear(self.after, self.after*2) self.linear_o_4_c = nn.Linear(1, 4)
self.linear_o_4_c = nn.Linear(self.after*2, self.after*4)
def init_hidden(self, batch_size): def init_hidden(self, batch_size):
hidden = torch.randn(self.nhidden_i, batch_size, self.n_out_i).to(device) hidden = torch.randn(self.nhidden_i, batch_size, self.n_out_i).to(device)
cell = torch.randn(self.nhidden_i, batch_size, self.n_out_i).to(device) cell = torch.randn(self.nhidden_i, batch_size, self.n_out_i).to(device)
return (hidden, cell) return (hidden, cell)
def forward(self, x0): def forward(self, x0):
self.hidden = self.init_hidden(x0.size(0)) self.hidden = self.init_hidden(x0.size(0))
x = self.lstm(x0, self.hidden)[0].reshape(x0.shape[0], -1) x = self.lstm(x0, self.hidden)[0].reshape(x0.shape[0], -1)
x = self.bn1(x) x = self.bn1(x)
# x = F.relu(x) # x = F.relu(x)
x = F.relu(self.linear_o_1(x)) x = F.relu(self.linear_o_1(x))
# x = self.ln1(x) # x = self.ln1(x)
x = F.dropout(x, 0.2, training=self.training) x = F.dropout(x, 0.2, training=self.training)
x = F.relu(self.linear_o_2(x)) x = F.relu(self.linear_o_2(x))
x = F.dropout(x, 0.2,