NNForecaster Training and Tuning

This tutorial shows how to utilize the built-in forecaster training and tuning functionalities of CommonPower.

%load_ext autoreload
%autoreload 2

from datetime import timedelta
import pathlib
from ray import tune as ray_tune, train as ray_train

from ray.tune.schedulers import AsyncHyperBandScheduler
from ray.tune.stopper import TrialPlateauStopper, CombinedStopper, MaximumIterationStopper
from ray.tune.search.hyperopt import HyperOptSearch
from ray.air.integrations.wandb import WandbLoggerCallback
from sklearn.preprocessing import MinMaxScaler

from commonpower.data_forecasting.nn_forecasting.config import ParameterSpace, TrainConfig
from commonpower.data_forecasting.nn_forecasting.eval_metrics import RootMeanSquaredError, MeanAbsolutePercentageError, MeanAbsoluteError
from commonpower.data_forecasting.nn_forecasting.nn_forecasting import NNForecaster
from commonpower.data_forecasting.data_sources import CSVDataSource
from commonpower.data_forecasting.nn_forecasting.data_splitting import SimpleFractionalSplit, DatePeriodFractionalSplit
from commonpower.data_forecasting.nn_forecasting.dataset_wrappers import NStepAhead
from commonpower.data_forecasting.nn_forecasting.models import *
from commonpower.data_forecasting.nn_forecasting.transform import SklearnScalerTransform

from commonpower.data_forecasting.nn_forecasting.nn_tuner import tune

experiments_dir = pathlib.Path().absolute()
data_path = (experiments_dir / 'data').resolve()
results_dir = (experiments_dir / 'results').resolve()
tuner_dir = (experiments_dir / 'tuner').resolve()

horizon = timedelta(hours=24)
frequency = timedelta(minutes=60)
look_back = timedelta(hours=24)

data_source = CSVDataSource(
    data_path / '1-LV-rural2--1-sw' / 'LoadProfile.csv',
    delimiter=";",
    datetime_format="%d.%m.%Y %H:%M",
    rename_dict={"time": "t", "H0-A_pload": "p"},
    auto_drop=True,
    resample=frequency)

Configure and Tune

train_config = TrainConfig(
    forecaster=NNForecaster(
        targets=["p"],
        model_class=SimpleMLP,
        frequency=frequency,
        feature_transform=SklearnScalerTransform(MinMaxScaler()),
        target_transform=SklearnScalerTransform(MinMaxScaler()),
    ),
    data_source=data_source,
    dataset_wrapper_class=NStepAhead,
    dataset_split_class=DatePeriodFractionalSplit,
    parameter_space=ParameterSpace(
        model={
            "hidden_dims": ray_tune.choice([[64, 256, 512, 512, 256, 64]]),
            "n_features": 1,
            "n_targets": 1,
            "n_lookback": ray_tune.choice([24]),
            "n_ahead": ray_tune.choice([6]),
            "dropout_p": ray_tune.choice([0.5]),
        },
        optimizer={
            "lr": 1e-3
        },
        data_loader={
            "batch_size": ray_tune.choice([128]),
        },
        dataset_wrapper={},
        dataset_split={
            "train_fraction": 0.8,
            "val_fraction": 0.2
        },
    ),
    eval_metrics=[RootMeanSquaredError(), MeanAbsoluteError(), MeanAbsolutePercentageError()],
    device='cpu'
)

run_config = ray_train.RunConfig(
    stop=CombinedStopper(
        MaximumIterationStopper(max_iter=1000),
        #TrialPlateauStopper(metric=RootMeanSquaredError().name, std=0.0001, num_results=5, grace_period=5),
    ),
    callbacks=[WandbLoggerCallback(project="commonpower_forecaster_tuning", mode="online", config=train_config.model_dump())],
    checkpoint_config=ray_train.CheckpointConfig(
        num_to_keep=10,
        checkpoint_score_attribute=RootMeanSquaredError().name,
        checkpoint_score_order="min"
    ),
    storage_path=tuner_dir,
)

tune_config = ray_tune.TuneConfig(
    metric=RootMeanSquaredError().name,
    mode='min',
    scheduler=None, #AsyncHyperBandScheduler(grace_period=500),
    num_samples=4,
    max_concurrent_trials=16,
    trial_dirname_creator=lambda trial: f"{trial.trial_id}",
)

tuned_forecaster, result_checkpoint = tune(
    train_config,
    tune_config,
    run_config,
    results_dir
)

Load from checkpoint and use

from commonpower.data_forecasting.base import DataProvider

forecaster = NNForecaster(
        targets=["p"],
        model_class=SimpleMLP,
        frequency=frequency,
        horizon=horizon,
        feature_transform=SklearnScalerTransform.from_checkpoint(result_checkpoint, "feature", scaler=MinMaxScaler()),
        target_transform=SklearnScalerTransform.from_checkpoint(result_checkpoint, "target", scaler=MinMaxScaler()),
).with_model(SimpleMLP.from_checkpoint(result_checkpoint))

data_provider = DataProvider(
    data_source=data_source,
    forecaster=forecaster,
)

import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime

timestamps = pd.date_range(start=datetime(2016, 6, 29, 6), periods=int(horizon / frequency) + 1, freq=frequency)

true_data = data_source(timestamps[0], timestamps[-1])
forecast = data_provider.observe(timestamps[0])["p"]

plt.plot(true_data, label="true data")
plt.plot(forecast, label="forecast")
plt.xticks(range(len(true_data)), timestamps.strftime('%H:%M'), rotation=90)
plt.xlabel("time step")
plt.ylabel("active power [kW]")
plt.legend()
plt.show()