dnn

Deep learning helpers built on the existing fastai training backbone, with runnable examples from a seaborn dataset.

Setup

Utils

---

source

seed_everything

def seed_everything(
    seed:int=123
)->None:

Seed Python, NumPy, and PyTorch for reproducible experiments.

seed_everything()

def_device

'mps'

Example Data

import seaborn as sns
from sklearn.model_selection import StratifiedKFold

seed_everything(123)
df = sns.load_dataset("penguins").dropna(
    subset=["bill_length_mm", "bill_depth_mm", "flipper_length_mm", "body_mass_g", "species"]
).reset_index(drop=True)
feat_col = ["bill_length_mm", "bill_depth_mm", "flipper_length_mm", "body_mass_g"]
target_df = pd.get_dummies(df["species"], prefix="species", dtype=float)
target_col = target_df.columns.tolist()
df[target_col] = target_df
n_feature = len(feat_col)
n_target = len(target_col)
n_aa = len(target_col)
skf = StratifiedKFold(n_splits=3, shuffle=True, random_state=123)
splits = list(skf.split(df.index, df["species"]))
split0 = splits[0]
df.shape

(342, 10)

df[feat_col + ["species"] + target_col].head()

	bill_length_mm	bill_depth_mm	flipper_length_mm	body_mass_g	species	species_Adelie	species_Chinstrap	species_Gentoo
0	39.1	18.7	181.0	3750.0	Adelie	1.0	0.0	0.0
1	39.5	17.4	186.0	3800.0	Adelie	1.0	0.0	0.0
2	40.3	18.0	195.0	3250.0	Adelie	1.0	0.0	0.0
3	36.7	19.3	193.0	3450.0	Adelie	1.0	0.0	0.0
4	39.3	20.6	190.0	3650.0	Adelie	1.0	0.0	0.0

Dataset

---

source

GeneralDataset

def GeneralDataset(
    df:DataFrame, feat_col, target_col:NoneType=None, A:int=23, dtype:type=float32
)->None:

An abstract class representing a :class:Dataset.

All datasets that represent a map from keys to data samples should subclass it. All subclasses should overwrite :meth:__getitem__, supporting fetching a data sample for a given key. Subclasses could also optionally overwrite :meth:__len__, which is expected to return the size of the dataset by many :class:~torch.utils.data.Sampler implementations and the default options of :class:~torch.utils.data.DataLoader. Subclasses could also optionally implement :meth:__getitems__, for speedup batched samples loading. This method accepts list of indices of samples of batch and returns list of samples.

.. note:: :class:~torch.utils.data.DataLoader by default constructs an index sampler that yields integral indices. To make it work with a map-style dataset with non-integral indices/keys, a custom sampler must be provided.

ds = GeneralDataset(df, feat_col, target_col, A=n_aa)
xb, yb = next(iter(DataLoader(ds, batch_size=8, shuffle=True)))
len(ds), xb.shape, yb.shape

(342, torch.Size([8, 4]), torch.Size([8, 3, 1]))

Models

MLP

---

source

MLP

def MLP(
    num_features:int, num_targets:int, hidden_units:list=[512, 218], dp:float=0.2
):

Feed-forward model for tabular inputs.

mlp = MLP(n_feature, n_target)
mlp(xb).shape

torch.Size([8, 3])

CNN1D

---

source

lin_wn

def lin_wn(
    ni, nf, dp:float=0.1, act:type=SiLU
):

Weight-normalized linear block.

lin_wn(10, 3)

Sequential(
  (0): BatchNorm1d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (1): Dropout(p=0.1, inplace=False)
  (2): ParametrizedLinear(
    in_features=10, out_features=3, bias=True
    (parametrizations): ModuleDict(
      (weight): ParametrizationList(
        (0): _WeightNorm()
      )
    )
  )
  (3): SiLU()
)

---

source

conv_wn

def conv_wn(
    ni, nf, ks:int=3, stride:int=1, padding:int=1, dp:float=0.1, act:type=ReLU
):

Weight-normalized convolution block.

---

source

CNN1D

def CNN1D(
    ni, nf, amp_scale:int=16
):

Base class for all neural network modules.

Your models should also subclass this class.

Modules can also contain other Modules, allowing them to be nested in a tree structure. You can assign the submodules as regular attributes::

import torch.nn as nn
import torch.nn.functional as F

class Model(nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.conv1 = nn.Conv2d(1, 20, 5)
        self.conv2 = nn.Conv2d(20, 20, 5)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        return F.relu(self.conv2(x))

Submodules assigned in this way will be registered, and will also have their parameters converted when you call :meth:to, etc.

.. note:: As per the example above, an __init__() call to the parent class must be made before assignment on the child.

:ivar training: Boolean represents whether this module is in training or evaluation mode. :vartype training: bool

---

source

init_weights

def init_weights(
    m, leaky:float=0.0
):

Initialize convolution layers with Kaiming normal weights.

cnn = CNN1D(n_feature, n_target).apply(init_weights)
cnn(xb).shape

torch.Size([8, 3])

Wrapper

---

source

PSSM_model

def PSSM_model(
    n_features, n_targets, A:int=23, model:str='MLP'
):

Base class for all neural network modules.

Your models should also subclass this class.

Modules can also contain other Modules, allowing them to be nested in a tree structure. You can assign the submodules as regular attributes::

import torch.nn as nn
import torch.nn.functional as F

class Model(nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.conv1 = nn.Conv2d(1, 20, 5)
        self.conv2 = nn.Conv2d(20, 20, 5)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        return F.relu(self.conv2(x))

Submodules assigned in this way will be registered, and will also have their parameters converted when you call :meth:to, etc.

.. note:: As per the example above, an __init__() call to the parent class must be made before assignment on the child.

:ivar training: Boolean represents whether this module is in training or evaluation mode. :vartype training: bool

model = PSSM_model(n_feature, n_target, A=n_aa, model='MLP')
logits = model(xb)
logits.shape

torch.Size([8, 3, 1])

Loss

---

source

CE

def CE(
    logits:Tensor, target_probs:Tensor
):

Cross-entropy with soft labels.

CE(logits, yb)

tensor(1.3051, grad_fn=<MeanBackward0>)

Metrics

---

source

KLD

def KLD(
    logits:Tensor, target_probs:Tensor
):

Average KL divergence across positions between target_probs and softmax(logits).

KLD(logits, yb)

tensor(1.3051, grad_fn=<MeanBackward0>)

---

source

JSD

def JSD(
    logits:Tensor, target_probs:Tensor
):

Average Jensen-Shannon divergence across positions between target_probs and softmax(logits).

JSD(logits, yb)

tensor(0.3545, grad_fn=<MeanBackward0>)

Trainer

---

source

train_dl

def train_dl(
    df:DataFrame, feat_col, target_col, split, model_func, A:int=23, n_epoch:int=4, bs:int=32, lr:float=0.01,
    loss:function=CE, save:NoneType=None, sampler:NoneType=None, lr_find:bool=False
):

Train a deep learning model with the fastai learner stack.

get_mlp = lambda: PSSM_model(n_feature, n_target, A=n_aa, model='MLP')
target, pred = train_dl(
    df,
    feat_col,
    target_col,
    split0,
    model_func=get_mlp,
    A=n_aa,
    n_epoch=1,
    bs=16,
    lr=3e-3,
    save='model',
)
pred.head()

lr in training is 0.003

epoch	train_loss	valid_loss	KLD	JSD	time
0	0.740750	3.037915	3.037915	0.376177	00:00

	species_Adelie	species_Chinstrap	species_Gentoo
0	0.009204	0.009344	0.981452
3	0.063467	0.055438	0.881095
9	0.182922	0.156108	0.660971
11	0.294663	0.286624	0.418712
14	0.011959	0.011384	0.976657

Predict

---

source

predict_dl

def predict_dl(
    df, feat_col, target_col, model_func, model_pth, A:int=23, bs:int=512
):

Predict a dataframe given a deep learning model saved by fastai.

test_pred = predict_dl(
    df.iloc[split0[1]].copy(),
    feat_col,
    target_col,
    model_func=get_mlp,
    model_pth='model',
    A=n_aa,
)
test_pred

	species_Adelie	species_Chinstrap	species_Gentoo
0	9.204363e-03	9.344031e-03	0.981452
3	6.346702e-02	5.543802e-02	0.881095
9	1.829216e-01	1.561077e-01	0.660971
11	2.946635e-01	2.866240e-01	0.418712
14	1.195943e-02	1.138383e-02	0.976657
...	...	...	...
328	5.574878e-09	1.360372e-08	1.000000
334	2.227252e-10	7.630787e-10	1.000000
335	5.731530e-07	1.071595e-06	0.999998
339	5.872652e-10	1.706496e-09	1.000000
340	5.236147e-08	1.102807e-07	1.000000

114 rows × 3 columns

CV train

---

source

train_dl_cv

def train_dl_cv(
    df, feat_col, target_col, splits, model_func, A:int=23, save:str | None=None, kwargs:VAR_KEYWORD
):

Cross-validation training loop for deep learning models.

oof = train_dl_cv(
    df,
    feat_col,
    target_col,
    splits=splits,
    model_func=get_mlp,
    A=n_aa,
    n_epoch=1,
    bs=16,
    lr=3e-3,
)
oof.nfold.value_counts()

------fold0------
lr in training is 0.003

epoch	train_loss	valid_loss	KLD	JSD	time
0	0.713906	2.601878	2.601878	0.355676	00:00

------fold1------
lr in training is 0.003

epoch	train_loss	valid_loss	KLD	JSD	time
0	0.699992	2.666008	2.666008	0.354615	00:00

------fold2------
lr in training is 0.003

epoch	train_loss	valid_loss	KLD	JSD	time
0	0.633100	2.994696	2.994696	0.354643	00:00

nfold
0    114
1    114
2    114
Name: count, dtype: int64