Transformer for Univariate Time Series Forecasting¶
In this notebook, we build a transformer using pytorch to forecast \(\sin\) function as a time series.
import dataclasses
import math
import lightning as L
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import torch
from lightning.pytorch.callbacks.early_stopping import EarlyStopping
from torch import nn
from ts_dl_utils.datasets.pendulum import Pendulum, PendulumDataModule
from ts_dl_utils.evaluation.evaluator import Evaluator
from ts_dl_utils.naive_forecasters.last_observation import LastObservationForecaster
Data¶
We create a dataset that models a damped pendulum. The pendulum is modelled as a damped harmonic oscillator, i.e.,
where \(\theta(t)\) is the angle of the pendulum at time \(t\). The period \(p\) is calculated using
with \(L\) being the length of the pendulum and \(g\) being the surface gravity.
pen = Pendulum(length=10000)
df = pd.DataFrame(pen(100, 400, initial_angle=1, beta=0.000001))
Since the damping constant is very small, the data generated is mostly a sin wave.
_, ax = plt.subplots(figsize=(10, 6.18))
df.plot(x="t", y="theta", ax=ax)
<Axes: xlabel='t'>

Model¶
In this section, we create the transformer model.
Since we do not deal with future covariates, we do not need a decoder. In this example, we build a simple transformer that only contains attention in encoder.
@dataclasses.dataclass
class TSTransformerParams:
"""A dataclass that contains all
the parameters for the transformer model.
"""
d_model: int = 512
nhead: int = 8
num_encoder_layers: int = 6
dropout: int = 0.1
class PositionalEncoding(nn.Module):
"""Positional encoding to be added to
input embedding.
:param d_model: hidden dimension of the encoder
:param dropout: rate of dropout
:param max_len: maximum length of our positional
encoder. The encoder can not encode sequence
length longer than max_len.
"""
def __init__(
self,
d_model: int,
dropout: float = 0.1,
max_len: int = 5000,
):
super().__init__()
self.max_len = max_len
self.dropout = nn.Dropout(p=dropout)
position = torch.arange(max_len).unsqueeze(1)
div_term = torch.exp(
torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model)
)
pe = torch.zeros(max_len, d_model)
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
self.register_buffer("pe", pe)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
:param x: input embedded time series,
shape `[batch_size, seq_len, embedding_dim]`
"""
history_length = x.size(1)
x = x + self.pe[:history_length]
return self.dropout(x)
class TSTransformer(nn.Module):
"""Transformer for univaraite time series modeling.
:param history_length: the length of the input history.
:param horizon: the number of steps to be forecasted.
:param transformer_params: all the parameters.
"""
def __init__(
self,
history_length: int,
horizon: int,
transformer_params: TSTransformerParams,
):
super().__init__()
self.transformer_params = transformer_params
self.history_length = history_length
self.horizon = horizon
self.embedding = nn.Linear(1, self.transformer_params.d_model)
self.positional_encoding = PositionalEncoding(
d_model=self.transformer_params.d_model
)
encoder_layer = nn.TransformerEncoderLayer(
d_model=self.transformer_params.d_model,
nhead=self.transformer_params.nhead,
batch_first=True,
)
self.encoder = nn.TransformerEncoder(
encoder_layer, num_layers=self.transformer_params.num_encoder_layers
)
self.reverse_embedding = nn.Linear(self.transformer_params.d_model, 1)
self.decoder = nn.Linear(self.history_length, self.horizon)
@property
def transformer_config(self) -> dict:
"""all the param in dict format"""
return dataclasses.asdict(self.transformer_params)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
:param x: input historical time series,
shape `[batch_size, seq_len, n_var]`
"""
x = self.embedding(x)
x = self.positional_encoding(x)
encoder_state = self.encoder(x)
decoder_in = self.reverse_embedding(encoder_state).squeeze(-1)
return self.decoder(decoder_in)
Training¶
We use lightning to train our model.
Training Utilities¶
history_length_1_step = 100
horizon_1_step = 1
gap = 0
We will build a few utilities
- To be able to feed the data into our model, we build a class (
DataFrameDataset) that converts the pandas dataframe into a Dataset for pytorch. - To make the lightning training code simpler, we will build a LightningDataModule (
PendulumDataModule) and a LightningModule (TransformerForecaster).
class TransformerForecaster(L.LightningModule):
"""Transformer forecasting training, validation,
and prediction all collected in one class.
:param transformer: pre-defined transformer model
"""
def __init__(self, transformer: nn.Module):
super().__init__()
self.transformer = transformer
self.save_hyperparameters()
def configure_optimizers(self) -> torch.optim.Optimizer:
optimizer = torch.optim.SGD(self.parameters(), lr=1e-3)
return optimizer
def training_step(self, batch: tuple[torch.Tensor], batch_idx: int) -> torch.Tensor:
x, y = batch
y = y.squeeze(-1).type(self.dtype)
y_hat = self.transformer(x)
loss = nn.functional.mse_loss(y_hat, y)
self.log_dict({"train_loss": loss}, prog_bar=True)
return loss
def validation_step(
self, batch: tuple[torch.Tensor], batch_idx: int
) -> torch.Tensor:
x, y = batch
y = y.squeeze(-1).type(self.dtype)
y_hat = self.transformer(x)
loss = nn.functional.mse_loss(y_hat, y)
self.log_dict({"val_loss": loss}, prog_bar=True)
return loss
def predict_step(
self, batch: list[torch.Tensor], batch_idx: int
) -> tuple[torch.Tensor]:
x, y = batch
y = y.squeeze(-1).type(self.dtype)
y_hat = self.transformer(x)
return x, y_hat
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor]:
return x, self.transformer(x)
Data, Model and Training¶
DataModule¶
pdm_1_step = PendulumDataModule(
history_length=history_length_1_step,
horizon=horizon_1_step,
dataframe=df[["theta"]],
gap=gap,
)
LightningModule¶
ts_transformer_params_1_step = TSTransformerParams(
d_model=192, nhead=6, num_encoder_layers=1
)
ts_transformer_1_step = TSTransformer(
history_length=history_length_1_step,
horizon=horizon_1_step,
transformer_params=ts_transformer_params_1_step,
)
ts_transformer_1_step
TSTransformer(
(embedding): Linear(in_features=1, out_features=192, bias=True)
(positional_encoding): PositionalEncoding(
(dropout): Dropout(p=0.1, inplace=False)
)
(encoder): TransformerEncoder(
(layers): ModuleList(
(0): TransformerEncoderLayer(
(self_attn): MultiheadAttention(
(out_proj): NonDynamicallyQuantizableLinear(in_features=192, out_features=192, bias=True)
)
(linear1): Linear(in_features=192, out_features=2048, bias=True)
(dropout): Dropout(p=0.1, inplace=False)
(linear2): Linear(in_features=2048, out_features=192, bias=True)
(norm1): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
(norm2): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
(dropout1): Dropout(p=0.1, inplace=False)
(dropout2): Dropout(p=0.1, inplace=False)
)
)
)
(reverse_embedding): Linear(in_features=192, out_features=1, bias=True)
(decoder): Linear(in_features=100, out_features=1, bias=True)
)
transformer_forecaster_1_step = TransformerForecaster(transformer=ts_transformer_1_step)
transformer_forecaster_1_step
TransformerForecaster(
(transformer): TSTransformer(
(embedding): Linear(in_features=1, out_features=192, bias=True)
(positional_encoding): PositionalEncoding(
(dropout): Dropout(p=0.1, inplace=False)
)
(encoder): TransformerEncoder(
(layers): ModuleList(
(0): TransformerEncoderLayer(
(self_attn): MultiheadAttention(
(out_proj): NonDynamicallyQuantizableLinear(in_features=192, out_features=192, bias=True)
)
(linear1): Linear(in_features=192, out_features=2048, bias=True)
(dropout): Dropout(p=0.1, inplace=False)
(linear2): Linear(in_features=2048, out_features=192, bias=True)
(norm1): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
(norm2): LayerNorm((192,), eps=1e-05, elementwise_affine=True)
(dropout1): Dropout(p=0.1, inplace=False)
(dropout2): Dropout(p=0.1, inplace=False)
)
)
)
(reverse_embedding): Linear(in_features=192, out_features=1, bias=True)
(decoder): Linear(in_features=100, out_features=1, bias=True)
)
)
Trainer¶
logger_1_step = L.pytorch.loggers.TensorBoardLogger(
save_dir="lightning_logs", name="transformer_ts_1_step"
)
trainer_1_step = L.Trainer(
precision="64",
max_epochs=100,
min_epochs=5,
callbacks=[
EarlyStopping(monitor="val_loss", mode="min", min_delta=1e-7, patience=3)
],
logger=logger_1_step,
)
GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
Fitting¶
demo_x = list(pdm_1_step.train_dataloader())[0][0].type(
transformer_forecaster_1_step.dtype
)
demo_x.shape
torch.Size([32, 100, 1])
nn.Linear(
1,
ts_transformer_1_step.transformer_params.d_model,
dtype=transformer_forecaster_1_step.dtype,
)(demo_x).shape
torch.Size([32, 100, 192])
ts_transformer_1_step.encoder(ts_transformer_1_step.embedding(demo_x)).shape
torch.Size([32, 100, 192])
trainer_1_step.fit(model=transformer_forecaster_1_step, datamodule=pdm_1_step)
Retrieving Predictions¶
predictions_1_step = trainer_1_step.predict(
model=transformer_forecaster_1_step, datamodule=pdm_1_step
)
Naive Forecaster¶
trainer_naive_1_step = L.Trainer(precision="64")
lobs_forecaster_1_step = LastObservationForecaster(horizon=horizon_1_step)
lobs_1_step_predictions = trainer_naive_1_step.predict(
model=lobs_forecaster_1_step, datamodule=pdm_1_step
)
Evaluations¶
evaluator_1_step = Evaluator(step=0)
fig, ax = plt.subplots(figsize=(50, 6.18))
ax.plot(
evaluator_1_step.y_true(dataloader=pdm_1_step.predict_dataloader()),
"g-",
label="truth",
)
ax.plot(evaluator_1_step.y(predictions_1_step), "r--", label="predictions")
ax.plot(evaluator_1_step.y(lobs_1_step_predictions), "b-.", label="naive predictions")
plt.legend()
fig, ax = plt.subplots(figsize=(10, 6.18))
inspection_slice_length = 200
ax.plot(
evaluator_1_step.y_true(dataloader=pdm_1_step.predict_dataloader())[
:inspection_slice_length
],
"g-",
label="truth",
)
ax.plot(
evaluator_1_step.y(predictions_1_step)[:inspection_slice_length],
"r--",
label="predictions",
)
ax.plot(
evaluator_1_step.y(lobs_1_step_predictions)[:inspection_slice_length],
"b-.",
label="naive predictions",
)
plt.legend()
To quantify the results, we compute a few metrics.
pd.merge(
evaluator_1_step.metrics(predictions_1_step, pdm_1_step.predict_dataloader()),
evaluator_1_step.metrics(lobs_1_step_predictions, pdm_1_step.predict_dataloader()),
how="left",
left_index=True,
right_index=True,
suffixes=["_transformer", "_naive"],
)
Here SMAPE is better because of better forecasts for larger values
Forecasting (horizon=3)¶
Train a Model¶
history_length_m_step = 100
horizon_m_step = 3
pdm_m_step = PendulumDataModule(
history_length=history_length_m_step,
horizon=horizon_m_step,
dataframe=df[["theta"]],
gap=gap,
)
ts_transformer_params_m_step = TSTransformerParams(
d_model=192, nhead=6, num_encoder_layers=1
)
ts_transformer_m_step = TSTransformer(
history_length=history_length_m_step,
horizon=horizon_m_step,
transformer_params=ts_transformer_params_m_step,
)
ts_transformer_m_step
transformer_forecaster_m_step = TransformerForecaster(transformer=ts_transformer_m_step)
logger_m_step = L.pytorch.loggers.TensorBoardLogger(
save_dir="lightning_logs", name="transformer_ts_m_step"
)
trainer_m_step = L.Trainer(
precision="64",
max_epochs=100,
min_epochs=5,
callbacks=[
EarlyStopping(monitor="val_loss", mode="min", min_delta=1e-7, patience=3)
],
logger=logger_m_step,
)
trainer_m_step.fit(model=transformer_forecaster_m_step, datamodule=pdm_m_step)
predictions_m_step = trainer_m_step.predict(
model=transformer_forecaster_m_step, datamodule=pdm_m_step
)
Naive Forecaster¶
trainer_naive_m_step = L.Trainer(precision="64")
lobs_forecaster_m_step = LastObservationForecaster(horizon=horizon_m_step)
lobs_m_step_predictions = trainer_naive_m_step.predict(
model=lobs_forecaster_m_step, datamodule=pdm_m_step
)
Evaluations¶
evaluator_m_step = Evaluator(step=2, gap=gap)
fig, ax = plt.subplots(figsize=(10, 6.18))
ax.plot(
evaluator_m_step.y_true(dataloader=pdm_m_step.predict_dataloader()),
"g-",
label="truth",
)
ax.plot(evaluator_m_step.y(predictions_m_step), "r--", label="predictions")
ax.plot(evaluator_m_step.y(lobs_m_step_predictions), "b-.", label="naive predictions")
plt.legend()
fig, ax = plt.subplots(figsize=(10, 6.18))
for i in np.arange(0, 1000, 120):
evaluator_m_step.plot_one_sample(ax=ax, predictions=predictions_m_step, idx=i)
evaluator_m_step.metrics(predictions_m_step, pdm_m_step.predict_dataloader())
evaluator_m_step.metrics(lobs_m_step_predictions, pdm_m_step.predict_dataloader())
Investigations¶
We dive deeper into the intermediate results of the model.
from pathlib import Path
import pandas as pd
import plotly.express as px
from torch.utils.data import DataLoader
from ts_dl_utils.datasets.dataset import DataFrameDataset
load_from_checkpoint = (
Path(
# "lightning_logs/transformer_ts_1_step/version_9"
"lightning_logs/transformer_ts_1_step/version_7"
)
/ "checkpoints"
)
# load_from_checkpoint = Path(logger_1_step.log_dir) / "checkpoints"
load_from_checkpoint, logger_1_step.log_dir
list(load_from_checkpoint.iterdir())[0]
transformer_forecaster_1_step_re = TransformerForecaster.load_from_checkpoint(
# load_from_checkpoint / "checkpoints/epoch=11-step=5495.ckpt"
list(load_from_checkpoint.iterdir())[0]
)
transformer_forecaster_1_step_re
Visualize Embeddings of Intermediate Layers¶
def embedding_extractor(
forecaster: TransformerForecaster, x: torch.Tensor
) -> tuple[torch.Tensor]:
"""compute the embeddings based on the input
:param forecaster: the trained forecaster
:param x: input historical time series,
"""
forecaster.transformer.to(x.device)
x_embedding = forecaster.transformer.embedding(
x.type_as(forecaster.transformer.embedding.weight)
)
x_positional = forecaster.transformer.positional_encoding(x_embedding)
encoder_state = forecaster.transformer.encoder(x_positional)
reversed = forecaster.transformer.reverse_embedding(encoder_state).squeeze(-1)
return x_embedding, x_positional, encoder_state, reversed
def create_embedding_dataframe(
dr_result: torch.Tensor,
n_batches: int,
input_example: torch.Tensor,
n_components: int,
) -> pd.DataFrame:
dr_df = pd.DataFrame(
dr_result.detach().numpy(), columns=[f"DR_{i+1}" for i in range(n_components)]
)
dr_df["batch"] = sum(
[[i] * (len(dr_df) // n_batches) for i in range(n_batches)], []
)
dr_df["sample_idx"] = list(range(len(dr_df) // n_batches)) * n_batches
dr_df["input"] = np.concatenate(
input_example[:n_batches].detach().numpy().astype("float32")
)
dr_df = dr_df.merge(
pd.DataFrame(
input_example[:n_batches].detach()[:, 0, 0].numpy(),
columns=["batch_first_value"],
)
.reset_index()
.rename(columns={"index": "batch"}),
how="left",
on="batch",
)
return dr_df
Prepare input data for embedding visualization.
investigation_dl = DataLoader(
dataset=DataFrameDataset(
dataframe=df[["theta"]],
history_length=history_length_1_step,
horizon=horizon_1_step,
gap=gap,
),
batch_size=400,
shuffle=False,
)
investigation_dl
input_example = list(investigation_dl)[0][0]
input_example.shape
(embedding_example, positional_example, encoder_example, reversed_example) = (
embedding_extractor(transformer_forecaster_1_step, input_example)
)
(
input_example.shape,
embedding_example.shape,
positional_example.shape,
encoder_example.shape,
reversed_example.shape,
)
n_batches, seq_len, _ = input_example.shape
df_example = pd.DataFrame(
{
"input": input_example.squeeze(-1).detach().cpu().numpy().reshape(-1),
"reversed": reversed_example.detach().cpu().numpy().reshape(-1),
"sample": np.repeat(np.arange(n_batches), seq_len),
}
)
df_example
px.scatter(
df_example, x="input", y="reversed", color="sample", height=600, width=600
).update_layout(yaxis_scaleanchor="x")
import numpy as np
from torchdr import PCA, TSNE, UMAP
Latent space visualization
n_components = 2
dr_reversed_result = TSNE(
# dr_reversed_result = UMAP(
# dr_reversed_result = PCA(
# n_neighbors=30,
# backend='torch',
perplexity=30,
n_components=n_components,
).fit_transform(
np.concatenate(
[
input_example.squeeze(-1).numpy().astype("float32"),
reversed_example.detach().numpy().astype("float32"),
],
axis=0,
)
)
dr_reversed_df = pd.DataFrame(
dr_reversed_result, columns=[f"DR_{i+1}" for i in range(n_components)]
)
dr_reversed_df["type"] = ["input"] * (len(dr_reversed_df) // 2) + ["embedded"] * (
len(dr_reversed_df) - len(dr_reversed_df) // 2
)
dr_reversed_df["sample_idx"] = list(range(len(dr_reversed_df) // 2)) + list(
range(len(dr_reversed_df) // 2)
)
px.scatter(
dr_reversed_df,
x="DR_1",
y="DR_2",
# z="DR_3",
color="sample_idx",
# color="DR_4",
symbol="type",
title="Embedding of Input and Encoded Time Series",
height=600,
width=800,
).update_layout(legend=dict(itemsizing="constant", orientation="h", y=-0.2)).show()
px.scatter_3d(
dr_reversed_df,
x="DR_1",
y="DR_2",
z="DR_3",
color="sample_idx",
# color="DR_4",
symbol="type",
title="Embedding of Input and Encoded Time Series",
height=600,
width=800,
).update_layout(legend=dict(itemsizing="constant", orientation="h", y=-0.2)).show()
px.scatter_3d(
dr_reversed_df.loc[dr_reversed_df["type"] == "embedded"],
x="DR_1",
y="DR_2",
z="DR_3",
color="sample_idx",
# color="DR_4",
symbol="type",
title="Embedding of Input Time Series",
height=600,
width=800,
).update_layout(legend=dict(itemsizing="constant", orientation="h", y=-0.2)).show()
Embedding outout
embedding_example.detach()[0].shape, input_example.detach()[0].shape
embedding_result = UMAP(
# embedding_result = TSNE(
n_neighbors=30,
backend="torch",
# perplexity=30,
n_components=n_components,
).fit_transform(
# np.concatenate(
# embedding_example.detach().numpy().astype("float32")
# )
embedding_example.detach().reshape(-1, embedding_example.shape[-1])
)
embedding_result.shape
dr_embedding_df = create_embedding_dataframe(
dr_result=embedding_result,
n_batches=embedding_example.shape[0],
input_example=input_example,
n_components=n_components,
)
dr_embedding_df
px.scatter(
dr_embedding_df,
x="input",
y="DR_1",
# x='DR_1',
# y='DR_2',
# z='DR_3',
# color='sample_idx',
# symbol='type',
# color="batch_first_value",
# color="input",
title="Dimension Reduction for Embedding of Input Time Series",
height=600,
width=800,
).update_layout(legend=dict(itemsizing="constant", orientation="h", y=-0.2)).show()
px.scatter_3d(
dr_embedding_df,
x="DR_1",
y="DR_2",
# z='DR_3',
z="input",
# color='sample_idx',
# symbol='type',
# color="batch",
# color="batch_first_value",
color="input",
title="UMAP Embedding of Input Time Series",
height=600,
width=800,
).update_layout(legend=dict(itemsizing="constant", orientation="h", y=-0.2)).show()
px.scatter(
pd.DataFrame(input_example.detach()[:, 0, :].numpy()).reset_index(),
x="index",
y=0,
height=600,
width=800,
)
Positional encoding output
positional_example_sample_size = 100
# positional_result = UMAP(
positional_result = TSNE(
# n_neighbors=30, backend='torch',
perplexity=30,
n_components=n_components,
).fit_transform(
positional_example.detach()[:positional_example_sample_size].reshape(
-1, positional_example.shape[-1]
)
)
positional_result.shape
dr_positional_df = create_embedding_dataframe(
dr_result=positional_result,
n_batches=positional_example_sample_size,
input_example=input_example,
n_components=n_components,
)
dr_positional_df = pd.DataFrame(
positional_result.detach().numpy(),
columns=[f"DR_{i+1}" for i in range(n_components)],
)
dr_positional_df["batch"] = sum(
[
[i] * (len(dr_positional_df) // positional_example_sample_size)
for i in range(positional_example_sample_size)
],
[],
)
dr_positional_df["sample_idx"] = (
list(range(len(dr_positional_df) // positional_example_sample_size))
* positional_example_sample_size
)
dr_positional_df["input"] = np.concatenate(
input_example[:positional_example_sample_size].detach().numpy().astype("float32")
)
dr_positional_df = dr_positional_df.merge(
pd.DataFrame(
input_example[:positional_example_sample_size].detach()[:, 0, 0].numpy(),
columns=["batch_first_value"],
)
.reset_index()
.rename(columns={"index": "batch"}),
how="left",
on="batch",
)
dr_positional_df
px.scatter_3d(
dr_positional_df,
x="DR_1",
y="DR_2",
z="input",
# color='sample_idx',
# symbol='type',
# color="batch",
color="batch_first_value",
title="UMAP Embedding of Positional Encoded Time Series",
height=600,
width=800,
).update_layout(legend=dict(itemsizing="constant", orientation="h", y=-0.2)).show()
px.scatter(
dr_positional_df,
x="DR_1",
y="DR_2",
# z='input',
# symbol='type',
color="batch",
# color="input",
# color="batch_first_value",
title="UMAP Embedding of Positional Encoded Time Series",
height=600,
width=800,
).update_layout(legend=dict(itemsizing="constant", orientation="h", y=-0.2)).show()
px.scatter(
dr_positional_df,
x="input",
y="DR_1",
color="batch_first_value",
)
Encoder output
# encoder_result = TSNE(
encoder_result = PCA(
# n_neighbors=30, backend='torch',
# perplexity=30,
n_components=n_components
).fit_transform(
encoder_example.detach()[:positional_example_sample_size].reshape(
-1, encoder_example.shape[-1]
)
)
encoder_result.shape
dr_encoder_df = create_embedding_dataframe(
dr_result=encoder_result,
# n_batches=encoder_example.shape[0],
n_batches=positional_example_sample_size,
input_example=input_example,
n_components=n_components,
)
px.scatter_3d(
dr_encoder_df,
x="DR_1",
y="DR_2",
z="input",
# color='sample_idx',
# symbol='type',
# color="batch",
color="batch_first_value",
title="UMAP Embedding of Encoder Encoded Time Series",
height=600,
width=800,
).update_layout(legend=dict(itemsizing="constant", orientation="h", y=-0.2)).show()
px.scatter(
dr_encoder_df,
x="input",
y="DR_1",
color="batch_first_value",
)