Skip to content

Random Forest Playground

Outline

  1. Generate data of specific functions
  2. Fit the functions using ensemble methods
  3. Analyze the trees
import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd
import sklearn.tree as _tree
from sklearn.ensemble import RandomForestRegressor
from sklearn.feature_selection import (
    SelectFromModel,
    SelectKBest,
    chi2,
    mutual_info_regression,
)
from sklearn.model_selection import (
    GridSearchCV,
    RandomizedSearchCV,
    cross_val_score,
    learning_curve,
    train_test_split,
    validation_curve,
)
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeRegressor
mpl.rcParams["axes.unicode_minus"] = False
from random import random
import numpy as np
import seaborn as sns
from joblib import dump, load

Model

Components

# Number of trees in random forest
n_estimators = [int(x) for x in np.linspace(1000, 4000, 11)]
# Number of features to consider at every split
max_features = ["auto", "sqrt"]
# Maximum number of levels in tree
max_depth = [int(x) for x in range(10, 30, 2)]
max_depth.append(None)
# Minimum number of samples required to split a node
min_samples_split = [0.001, 0.01, 0.02, 0.05, 0.1, 0.2]
# Minimum number of samples required at each leaf node
min_samples_leaf = [10, 20, 30, 40, 50]
# Method of selecting samples for training each tree
bootstrap = [True, False]


rf_random_grid = {
    "rf__n_estimators": n_estimators,
    #     "rf__max_features": max_features,
    "rf__max_depth": max_depth,
    "rf__min_samples_split": min_samples_split,
    "rf__min_samples_leaf": min_samples_leaf,
    "rf__bootstrap": bootstrap,
}

rf = RandomForestRegressor(random_state=42, oob_score=True)

##########

pipeline_steps = [
    ("rf", rf),
]

pipeline = Pipeline(pipeline_steps)
def pred_true_comparison_plot(dataframe, ax, pred_sample=100):
    sns.scatterplot(
        dataframe, x="x", y="y", ax=ax, label="y", marker=".", ec="face", s=5
    )

    sns.scatterplot(
        dataframe.sample(100),
        x="x",
        y="y_pred",
        ax=ax,
        label="y_pred",
        marker="+",
        s=100,
        linewidth=2,
    )

    return ax


def predictions_each_estimators(x, rf_model):
    preds = []
    for i in x:
        i_preds = []
        for est in rf_model.best_estimator_["rf"].estimators_:
            i_preds.append(est.predict([[i]]).tolist())
        i_preds = sum(i_preds, [])
        preds.append(i_preds)

    return {"x": pd.DataFrame(x, columns=["x"]), "preds": pd.DataFrame(preds)}

Data without Noise

X_sin = [6 * random() for i in range(10000)]
y_sin = np.sin(X_sin)

X_sin_test = [6 * random() for i in range(10000)]
y_sin_test = np.sin(X_sin_test)
df_sin = pd.DataFrame(
    {
        "x": X_sin,
        "y": y_sin,
    }
)
model = RandomizedSearchCV(
    pipeline, cv=10, param_distributions=rf_random_grid, verbose=3, n_jobs=-1
)
model.fit(df_sin[["x"]], df_sin["y"].values.ravel())
sin_score = model.score(df_sin[["x"]], df_sin["y"].values.ravel())
model.best_params_
# dump(model, "reports/rf_sin.joblib")
fig, ax = plt.subplots(figsize=(10 * 10, 4 * 10))
_tree.plot_tree(model.best_estimator_["rf"].estimators_[0], fontsize=7)

Plot out the result

df_sin["y_pred"] = model.predict(df_sin[["x"]])
df_sin
fig, ax = plt.subplots(figsize=(10, 6.18))

pred_true_comparison_plot(df_sin, ax)
ax.set_title(f"Random Forest on Sin Data; $R^2$ Score: {sin_score:0.2f}")
plt.legend()

Plot out the boxplots of each data point

est_sample_skip = 100
sin_est_pred = predictions_each_estimators(sorted(X_sin_test)[::est_sample_skip], model)
df_sin_est_quantiles = pd.merge(
    sin_est_pred["x"],
    sin_est_pred["preds"].quantile(q=[0.75, 0.25], axis=1).T,
    how="left",
    left_index=True,
    right_index=True,
)

df_sin_est_quantiles["boxsize"] = (
    df_sin_est_quantiles[0.75] - df_sin_est_quantiles[0.25]
)
fig, ax = plt.subplots(figsize=(10, 1.5 * 6.18))
fig_skip = 5

ax.violinplot(
    sin_est_pred["preds"].values.tolist()[::fig_skip],
    positions=sin_est_pred["x"].x.tolist()[::fig_skip],
)

sns.lineplot(df_sin, x="x", y="y", ax=ax, label="y")

plt.xticks([])
# ax.yaxis.set_major_locator(mpl.ticker.FixedLocator(range(10)))
# ax.set_xticklabels([f"{i:0.2f}" for i in sin_est_pred["x"].x])

ax.set_title(
    "Violin Plot for All Predictions of Each Tree in a Random Forest on Some Sin Data Points"
)
fig, ax = plt.subplots(figsize=(10, 2 * 6.18))
fig_skip = 5

ax.boxplot(
    sin_est_pred["preds"].values.tolist()[::fig_skip],
    positions=sin_est_pred["x"].x.tolist()[::fig_skip],
)
sns.lineplot(df_sin, x="x", y="y", ax=ax, label="y")

ax.set_xticklabels([f"{i:0.2f}" for i in sin_est_pred["x"].x[::fig_skip]])

ax.set_title("Box Plot for Tree Predictions on Random Forest on Sin Data")
df_sin_est_quantiles
fig, ax = plt.subplots(figsize=(10, 6.18))

sns.barplot(df_sin_est_quantiles, x="x", y="boxsize")

ax.set_xticklabels([f"{i:0.2f}" for i in sin_est_pred["x"].x])
fig, ax = plt.subplots(figsize=(10, 6.18))

sns.histplot(df_sin_est_quantiles.boxsize, ax=ax)
ax.set_yscale("log")
ax.set_xscale("log")

Data with Noise

X_sin_noise = np.array([6 * random() for i in range(10000)])
y_sin_noise = np.array([i + 0.1 * (random() - 0.5) for i in np.sin(X_sin_noise)])


df_sin_noise = pd.DataFrame({"x": X_sin_noise, "y": y_sin_noise})
fig, ax = plt.subplots(figsize=(10, 6.18))

sns.lineplot(df_sin_noise, x="x", y="y", ax=ax, label="y")
model_noise = RandomizedSearchCV(
    pipeline, cv=10, param_distributions=rf_random_grid, verbose=3, n_jobs=-1
)
model_noise.fit(df_sin_noise[["x"]], df_sin_noise["y"].values.ravel())
sin_noise_score = model_noise.score(df_sin_noise[["x"]], df_sin_noise["y"])
model_noise.best_params_
# dump(model_noise, "reports/rf_sin_noise.joblib")
fig, ax = plt.subplots(figsize=(9 * 10, 4 * 10))
_tree.plot_tree(model_noise.best_estimator_["rf"].estimators_[0], fontsize=7)
df_sin_noise["y_pred"] = model_noise.predict(df_sin_noise[["x"]])
fig, ax = plt.subplots(figsize=(10, 6.18))

pred_true_comparison_plot(df_sin_noise, ax)

ax.set_title(
    f"Random Forest on Sin Data with Noise; Test $R^2$ Score: {sin_noise_score:0.2f}"
)

plt.legend()
sin_noise_est_pred = predictions_each_estimators(
    sorted(X_sin_noise)[::100], model_noise
)
df_sin_noise_est_quantiles = pd.merge(
    sin_noise_est_pred["x"],
    sin_noise_est_pred["preds"].quantile(q=[0.75, 0.25], axis=1).T,
    how="left",
    left_index=True,
    right_index=True,
)

df_sin_noise_est_quantiles["boxsize"] = (
    df_sin_noise_est_quantiles[0.75] - df_sin_noise_est_quantiles[0.25]
)
fig, ax = plt.subplots(figsize=(10, 1.5 * 6.18))
fig_skip = 5

ax.violinplot(
    sin_noise_est_pred["preds"].values.tolist()[::fig_skip],
    positions=sin_noise_est_pred["x"].x.tolist()[::fig_skip],
)

sns.scatterplot(
    df_sin_noise, x="x", y="y", ax=ax, label="y", marker=".", ec="face", s=1
)

plt.xticks([])

ax.set_title(
    "Violin Plot for All Predictions of Each Tree in a Random Forest on Some Sin Data Points"
)
fig, ax = plt.subplots(figsize=(10, 2 * 6.18))
fig_skip = 5

ax.boxplot(
    sin_noise_est_pred["preds"].values.tolist()[::fig_skip],
    positions=sin_noise_est_pred["x"].x.tolist()[::fig_skip],
)
sns.scatterplot(
    df_sin_noise, x="x", y="y", ax=ax, label="y", marker=".", ec="face", s=1
)


ax.set_xticklabels([f"{i:0.2f}" for i in sin_noise_est_pred["x"].x[::fig_skip]])

ax.set_title("Box Plot for Tree Predictions on Random Forest on Sin Data")
df_sin_noise_est_quantiles["model"] = "with_noise"
df_sin_est_quantiles["model"] = "no_noise"

df_quantiles = pd.concat(
    [
        df_sin_est_quantiles[["model", "boxsize"]],
        df_sin_noise_est_quantiles[["model", "boxsize"]],
    ]
)
fig, ax = plt.subplots(figsize=(10, 6.18))

sns.boxplot(df_quantiles, x="boxsize", y="model", ax=ax)
fig, ax = plt.subplots(figsize=(10, 6.18))

sns.histplot(
    df_sin_noise_est_quantiles.boxsize,
    #     bins=20,
    ax=ax,
    kde=True,
    label="with Noise",
    stat="probability",
    binwidth=0.002,
    binrange=(0, 0.07),
)

sns.histplot(
    df_sin_est_quantiles.boxsize,
    #     bins=20,
    ax=ax,
    kde=True,
    label="without Noise",
    stat="probability",
    binwidth=0.002,
    binrange=(0, 0.07),
)

plt.legend()
fig, ax = plt.subplots(figsize=(10, 6.18))

sns.kdeplot(
    df_sin_noise_est_quantiles.boxsize,
    #     bins=20,
    ax=ax,
    #     hist=False,
    label="with Noise",
)

sns.kdeplot(
    df_sin_est_quantiles.boxsize,
    #     bins=20,
    ax=ax,
    #     hist=True,
    label="without Noise",
)
plt.legend()

Generalization Error

\[ P_{err} \leq \frac{\bar \rho (1-s^2) }{s^2}. \]
def generalization_error(rhob, s):
    res = rhob * (1 - s**2) / s**2

    if res > 1:
        res = 1

    return res
generalization_error(0.2, 0.8)
pe_data = [
    [generalization_error(rhob, s) for s in np.linspace(0.01, 0.1, 10)]
    for rhob in np.linspace(0.01, 0.1, 10)
]

pe_data_s_label = [s for s in np.linspace(0.01, 0.1, 10)]

pe_data_rhob_label = [rhob for rhob in np.linspace(0.01, 0.1, 10)]

fig, ax = plt.subplots(figsize=(10, 10))

sns.heatmap(pe_data, center=0, ax=ax)

ax.set_xlabel("s")
ax.set_ylabel("correlation")
ax.set_xticklabels([f"{i:0.2f}" for i in pe_data_s_label])
ax.set_yticklabels([f"{i:0.2f}" for i in pe_data_rhob_label])
temp_space = np.linspace(0.1, 1, 91)

pe_data = [[generalization_error(rhob, s) for s in temp_space] for rhob in temp_space]

pe_data_s_label = [s for s in temp_space]

pe_data_rhob_label = [rhob for rhob in temp_space]

fig, ax = plt.subplots(figsize=(12, 10))

sns.heatmap(pe_data, center=0, ax=ax)

ax.set_xlabel("s")
ax.set_ylabel("correlation")

ax.set_xticklabels([f"{i:0.2f}" for i in (ax.get_xticks() + 0.1) / 100])
ax.set_yticklabels([f"{i:0.2f}" for i in (ax.get_yticks() + 0.1) / 100])

for label in ax.xaxis.get_ticklabels()[::2]:
    label.set_visible(False)

for label in ax.yaxis.get_ticklabels()[::2]:
    label.set_visible(False)


ax.set_title("Upper Limit of Generalization Error of Random Forest")