Skip to content

experiment

Entry points for Hydra examples: the generic run_experiment lifecycle plus the training and inference tasks. For the guide, see Running an experiment.

mlkit.util.experiment

InferRun dataclass

Bases: RunContext

Inference state with trained weights loaded.

model_bundle["model"] is on accelerator.device in eval mode. train_cfg is the config saved with the checkpoint, or None for checkpoints without one. The datasets are None without get_dataset. outputs holds what infer returned.

Source code in src/mlkit/util/experiment/infer.py
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
@dataclass(kw_only=True)
class InferRun(RunContext):
    """Inference state with trained weights loaded.

    ``model_bundle["model"]`` is on ``accelerator.device`` in eval mode. ``train_cfg``
    is the config saved with the checkpoint, or ``None`` for checkpoints without one.
    The datasets are ``None`` without ``get_dataset``. ``outputs`` holds what
    ``infer`` returned.
    """

    model_bundle: dict[str, Any]
    checkpoint_dir: str
    train_cfg: DictConfig | None = None
    train_set: Any = None
    val_set: Any = None
    test_set: Any = None
    outputs: Any = None

InferSpec dataclass

Bases: Task[InferRun]

Load trained weights, then run inference hooks.

Weights come from cfg.infer.checkpoint_dir when set, otherwise from the latest checkpoint under the training save directory for run_suffix. Use the same run_suffix and training overrides as the training run. get_model builds the model as for training; its "train_hook_handlers" are disabled. The global main logs a warning for every model-side config value that differs from the config saved with the checkpoint.

Hook timing (relative to accelerator.end_training()): infer: every rank, while the process group is alive (sharded inference, collectives). Its return value is stored in run.outputs. on_finish: global main only, inside :meth:finish after end_training (sampling, plots; single-process work).

Both hooks run under accelerator.autocast(), so infer.mixed_precision applies to them.

Source code in src/mlkit/util/experiment/infer.py
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
@dataclass(kw_only=True)
class InferSpec(Task[InferRun]):
    """Load trained weights, then run inference hooks.

    Weights come from ``cfg.infer.checkpoint_dir`` when set, otherwise from the
    latest checkpoint under the training save directory for ``run_suffix``. Use the
    same ``run_suffix`` and training overrides as the training run. ``get_model``
    builds the model as for training; its ``"train_hook_handlers"`` are disabled.
    The global main logs a warning for every model-side config value that differs
    from the config saved with the checkpoint.

    Hook timing (relative to ``accelerator.end_training()``):
        ``infer``: every rank, while the process group is alive (sharded inference,
            collectives). Its return value is stored in ``run.outputs``.
        ``on_finish``: global main only, inside :meth:`finish` after ``end_training``
            (sampling, plots; single-process work).

    Both hooks run under ``accelerator.autocast()``, so ``infer.mixed_precision``
    applies to them.
    """

    section: ClassVar[str] = "infer"

    get_model: GetModelFn
    get_dataset: GetDatasetFn | None = None
    infer: Callable[[InferRun], Any] | None = None
    on_finish: Callable[[InferRun], None] | None = None
    run_suffix: str = ""

    def run_name(self, cfg: DictConfig) -> str:
        from mlkit.util.utils_for_main import get_run_name

        return get_run_name(cfg, suffix=f"{self.run_suffix}-infer")

    def run(self, ctx: RunContext) -> InferRun:
        import os

        from accelerate.utils import SAFE_WEIGHTS_NAME
        from safetensors.torch import load_model

        cfg = ctx.cfg
        checkpoint_dir = self._checkpoint_dir(cfg)
        train_cfg = _load_train_cfg(checkpoint_dir)
        if ctx.logger is not None:
            _warn_on_config_drift(ctx.logger, checkpoint_dir, train_cfg, cfg)

        model_bundle = build_model_bundle(self.get_model, cfg, disable_train_hooks=True)
        model = model_bundle["model"]
        load_model(model, os.path.join(checkpoint_dir, SAFE_WEIGHTS_NAME))
        model.to(ctx.accelerator.device).eval()

        datasets = self.get_dataset(cfg) if self.get_dataset is not None else (None, None, None)
        train_set, val_set, test_set = datasets
        run = InferRun(
            cfg=cfg,
            accelerator=ctx.accelerator,
            logger=ctx.logger,
            reporter=ctx.reporter,
            model_bundle=model_bundle,
            checkpoint_dir=checkpoint_dir,
            train_cfg=train_cfg,
            train_set=train_set,
            val_set=val_set,
            test_set=test_set,
        )
        if self.infer is not None:
            with ctx.accelerator.autocast():
                run.outputs = self.infer(run)
        return run

    def finish(self, run: InferRun) -> None:
        if self.on_finish is not None:
            with run.accelerator.autocast():
                self.on_finish(run)

    def _checkpoint_dir(self, cfg: DictConfig) -> str:
        from mlkit.pipeline.pipeline import latest_checkpoint_dir

        explicit = cfg.infer.get("checkpoint_dir")
        if explicit:
            return str(explicit)
        save_dir = checkpoint_root(cfg, self.run_suffix)
        if save_dir == "":
            raise FileNotFoundError(
                "train.save_dir is empty, so training saved no checkpoint; set infer.checkpoint_dir"
            )
        checkpoint_dir = latest_checkpoint_dir(save_dir)
        if checkpoint_dir is None:
            raise FileNotFoundError(f"No checkpoint under {save_dir}; train first or set infer.checkpoint_dir")
        return checkpoint_dir

RunContext dataclass

Process state that :func:run_experiment hands to a :class:Task.

logger and reporter are None off the global main process.

Source code in src/mlkit/util/experiment/runner.py
30
31
32
33
34
35
36
37
38
39
40
@dataclass(kw_only=True)
class RunContext:
    """Process state that :func:`run_experiment` hands to a :class:`Task`.

    ``logger`` and ``reporter`` are ``None`` off the global main process.
    """

    cfg: DictConfig
    accelerator: Accelerator
    logger: logging.Logger | None
    reporter: Reporter | None

Task

Bases: ABC

One job that :func:run_experiment runs inside a single process group.

cfg[section] supplies seed and mixed_precision.

Hook timing (relative to accelerator.end_training()): run: every rank, while the process group is alive. finish: global main only, after end_training, while the reporter is still open.

Source code in src/mlkit/util/experiment/runner.py
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
class Task[R: RunContext](ABC):
    """One job that :func:`run_experiment` runs inside a single process group.

    ``cfg[section]`` supplies ``seed`` and ``mixed_precision``.

    Hook timing (relative to ``accelerator.end_training()``):
        ``run``: every rank, while the process group is alive.
        ``finish``: global main only, after ``end_training``, while the reporter
            is still open.
    """

    section: ClassVar[str]

    @abstractmethod
    def run_name(self, cfg: DictConfig) -> str:
        """Return the tracked run name. Called on the global main only."""

    def resume_run_id(self, cfg: DictConfig, backend: str) -> str | None:
        """Return the tracked run to continue, or ``None`` for a new run.

        Args:
            cfg: experiment config.
            backend: reporter that is about to be opened.
        """
        del cfg, backend
        return None

    @abstractmethod
    def run(self, ctx: RunContext) -> R:
        """Do the work on every rank and return the state handed to :meth:`finish`."""

    def finish(self, run: R) -> None:
        """Do single-process work on the global main after the process group is gone."""
        del run

run_name(cfg) abstractmethod

Return the tracked run name. Called on the global main only.

Source code in src/mlkit/util/experiment/runner.py
56
57
58
@abstractmethod
def run_name(self, cfg: DictConfig) -> str:
    """Return the tracked run name. Called on the global main only."""

resume_run_id(cfg, backend)

Return the tracked run to continue, or None for a new run.

Parameters:

Name Type Description Default
cfg DictConfig

experiment config.

required
backend str

reporter that is about to be opened.

required
Source code in src/mlkit/util/experiment/runner.py
60
61
62
63
64
65
66
67
68
def resume_run_id(self, cfg: DictConfig, backend: str) -> str | None:
    """Return the tracked run to continue, or ``None`` for a new run.

    Args:
        cfg: experiment config.
        backend: reporter that is about to be opened.
    """
    del cfg, backend
    return None

run(ctx) abstractmethod

Do the work on every rank and return the state handed to :meth:finish.

Source code in src/mlkit/util/experiment/runner.py
70
71
72
@abstractmethod
def run(self, ctx: RunContext) -> R:
    """Do the work on every rank and return the state handed to :meth:`finish`."""

finish(run)

Do single-process work on the global main after the process group is gone.

Source code in src/mlkit/util/experiment/runner.py
74
75
76
def finish(self, run: R) -> None:
    """Do single-process work on the global main after the process group is gone."""
    del run

ConfigSnapshotCallback

Bases: BaseCallback

Write the resolved experiment config into each checkpoint before it is published.

The config is read at save time, so rewrites made before training (LR scheduler length) are included.

Source code in src/mlkit/util/experiment/train.py
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
class ConfigSnapshotCallback(BaseCallback):
    """Write the resolved experiment config into each checkpoint before it is published.

    The config is read at save time, so rewrites made before training (LR scheduler
    length) are included.
    """

    def __init__(self, cfg: DictConfig) -> None:
        self._cfg = cfg

    def on_event(self, event: CallbackEvent, *_args: object, **kwargs: object) -> None:
        import os

        from omegaconf import OmegaConf

        if event is not CallbackEvent.PRE_FINALIZE_SAVE:
            return
        directory = kwargs.get("checkpoint_dir")
        if isinstance(directory, str):
            OmegaConf.save(self._cfg, os.path.join(directory, CONFIG_SNAPSHOT_NAME), resolve=True)

TrainRun dataclass

Bases: RunContext

Training state built before pipeline.train().

accelerator is pipeline.accelerator: the instance that called prepare on the model, optimizer, dataloader, and scheduler (use it for save_state, unwrap_model, clip_grad_norm_, and training collectives). run_experiment owns a separate session Accelerator for process-group lifecycle only.

Teardown hooks receive the same instance after training; pipeline reflects post-train state.

Source code in src/mlkit/util/experiment/train.py
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
@dataclass(kw_only=True)
class TrainRun(RunContext):
    """Training state built before ``pipeline.train()``.

    ``accelerator`` is ``pipeline.accelerator``: the instance that called ``prepare``
    on the model, optimizer, dataloader, and scheduler (use it for ``save_state``,
    ``unwrap_model``, ``clip_grad_norm_``, and training collectives). ``run_experiment``
    owns a separate session ``Accelerator`` for process-group lifecycle only.

    Teardown hooks receive the same instance after training; ``pipeline`` reflects
    post-train state.
    """

    model_bundle: dict[str, Any]
    train_set: Any
    val_set: Any
    test_set: Any
    optimizer: Optimizer
    lr_scheduler: LRScheduler
    log_config: LogConfig
    training_config: TrainingConfig
    collate_fn: Callable[..., Any] | None
    pipeline: DistributedPipeline

TrainSpec dataclass

Bases: Task[TrainRun]

Train one model with the Accelerate pipeline.

get_model may return a bare model or a bundle dict with "model" plus optional "callbacks" (passed to the pipeline constructor, so they see POST_LOAD) and "train_hook_handlers". run_suffix is appended to both the run name and the checkpoint directory. Every published checkpoint also holds the resolved config as :data:CONFIG_SNAPSHOT_NAME.

Hook timing (relative to accelerator.end_training()): after_train: every rank, after pipeline.train() while the process group is still alive (optional second training, shared reporter). on_finish: global main only, inside :meth:finish after end_training.

Source code in src/mlkit/util/experiment/train.py
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
@dataclass(kw_only=True)
class TrainSpec(Task[TrainRun]):
    """Train one model with the Accelerate pipeline.

    ``get_model`` may return a bare model or a bundle dict with ``"model"`` plus
    optional ``"callbacks"`` (passed to the pipeline constructor, so they see
    ``POST_LOAD``) and ``"train_hook_handlers"``. ``run_suffix`` is appended to
    both the run name and the checkpoint directory. Every published checkpoint
    also holds the resolved config as :data:`CONFIG_SNAPSHOT_NAME`.

    Hook timing (relative to ``accelerator.end_training()``):
        ``after_train``: every rank, after ``pipeline.train()`` while the process
            group is still alive (optional second training, shared reporter).
        ``on_finish``: global main only, inside :meth:`finish` after ``end_training``.
    """

    section: ClassVar[str] = "train"

    get_model: GetModelFn
    get_dataset: GetDatasetFn
    get_collate: GetCollateFn
    get_optimizer: GetOptimizerFn | None = None
    after_train: Callable[[TrainRun], None] | None = None
    on_finish: Callable[[TrainRun], None] | None = None
    run_suffix: str = ""
    disable_train_hooks: bool = True

    def run_name(self, cfg: DictConfig) -> str:
        from mlkit.util.utils_for_main import get_run_name

        return get_run_name(cfg, suffix=self.run_suffix)

    def resume_run_id(self, cfg: DictConfig, backend: str) -> str | None:
        from mlkit.pipeline.pipeline import latest_checkpoint_dir
        from mlkit.util.report import resume_run_id

        return resume_run_id(latest_checkpoint_dir(checkpoint_root(cfg, self.run_suffix)), backend)

    def run(self, ctx: RunContext) -> TrainRun:
        run = self._materialize(ctx)
        run.pipeline.train()
        if self.after_train is not None and not run.pipeline.halted:
            self.after_train(run)
        return run

    def finish(self, run: TrainRun) -> None:
        if self.on_finish is not None:
            self.on_finish(run)

    def _materialize(self, ctx: RunContext) -> TrainRun:
        """Build model, data, optimizer, scheduler, and pipeline (no ``pipeline.train()``).

        May rewrite ``cfg`` for the LR scheduler before ``store_config``.
        """
        from omegaconf import OmegaConf

        from mlkit.pipeline.pipeline import LogConfig
        from mlkit.util.show import show_info
        from mlkit.util.utils_for_main import (
            get_learing_rate_scheduler,
            get_optimizer,
            get_train_class,
        )

        cfg = ctx.cfg
        model_bundle = build_model_bundle(self.get_model, cfg, disable_train_hooks=self.disable_train_hooks)
        model = model_bundle["model"]

        train_set, val_set, test_set = self.get_dataset(cfg)
        optimizer = (self.get_optimizer or get_optimizer)(model, cfg)
        lr_scheduler = get_learing_rate_scheduler(optimizer, ctx.accelerator, train_set, cfg)
        if ctx.accelerator.is_main_process:
            effective_config = cast(dict[str, Any], OmegaConf.to_container(cfg, resolve=True))
            if ctx.reporter is not None:
                ctx.reporter.store_config(effective_config)
            if ctx.logger is not None:
                ctx.logger.info("Config after scheduler rewrite:\n" + OmegaConf.to_yaml(cfg))
        show_info(model=model, optimizer=optimizer)

        log_config = LogConfig(**cfg.log)
        PipelineClass, TrainingConfigClass = get_train_class()
        training_config = TrainingConfigClass(**cfg.train)
        training_config.save_dir = checkpoint_root(cfg, self.run_suffix)

        print(training_config.__dict__)

        collate_fn = self.get_collate(cfg)
        callbacks = [*model_bundle.get("callbacks", []), ConfigSnapshotCallback(cfg)]
        if ctx.reporter is not None:
            callbacks.append(ctx.reporter.as_callback())
        pipeline = PipelineClass(
            model=model,
            train_dataset=train_set,
            eval_dataset=val_set,
            optimizers=(optimizer, lr_scheduler),
            training_config=training_config,
            log_config=log_config,
            collate_fn=collate_fn,
            logger=ctx.logger,
            seed=cfg.train.seed,
            callbacks=callbacks,
        )
        return TrainRun(
            cfg=cfg,
            accelerator=pipeline.accelerator,
            logger=ctx.logger,
            reporter=ctx.reporter,
            model_bundle=model_bundle,
            train_set=train_set,
            val_set=val_set,
            test_set=test_set,
            optimizer=optimizer,
            lr_scheduler=lr_scheduler,
            log_config=log_config,
            training_config=training_config,
            collate_fn=collate_fn,
            pipeline=pipeline,
        )

run_experiment(cfg, task)

Run task inside one Accelerate process group with logging and tracking.

The global main opens the log file and the reporter first; a failure there stops every rank before task.run. A failure inside task.run destroys the group without a barrier and skips task.finish. Otherwise every rank waits, the group is destroyed, a deferred rank-local failure is raised, and task.finish runs on the global main. A failure to destroy the group is logged, not raised.

Returns:

Type Description
R

The state returned by task.run on this rank.

Source code in src/mlkit/util/experiment/runner.py
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def run_experiment[R: RunContext](cfg: DictConfig, task: Task[R]) -> R:
    """Run ``task`` inside one Accelerate process group with logging and tracking.

    The global main opens the log file and the reporter first; a failure there stops
    every rank before ``task.run``. A failure inside ``task.run`` destroys the group
    without a barrier and skips ``task.finish``. Otherwise every rank waits, the group
    is destroyed, a deferred rank-local failure is raised, and ``task.finish`` runs on
    the global main. A failure to destroy the group is logged, not raised.

    Returns:
        The state returned by ``task.run`` on this rank.
    """

    import os
    from contextlib import ExitStack

    from accelerate import Accelerator
    from omegaconf import OmegaConf

    from mlkit.util.failure import clear_deferred_error, raise_deferred_error
    from mlkit.util.log import get_and_create_new_log_dir, get_logger
    from mlkit.util.report import ReporterContext
    from mlkit.util.seed import seed_everything

    settings = cfg[task.section]
    seed_everything(settings.seed)

    accelerator = Accelerator(mixed_precision=settings.mixed_precision)
    print(f"accelerator.device = {accelerator.device}")

    logger: logging.Logger | None = None
    reporter: Reporter | None = None

    def _reporter_open_failed(local_open_error: BaseException | None) -> bool:
        if accelerator.num_processes == 1:
            return local_open_error is not None
        if local_open_error is not None:
            accelerator.set_trigger()
        return bool(accelerator.check_trigger())

    clear_deferred_error()
    with ExitStack() as stack:
        local_open_error: BaseException | None = None
        if accelerator.is_main_process:
            try:
                log_dir = get_and_create_new_log_dir(cfg.log.log_dir, suffix=str(os.getpid()))
                logger = get_logger(name="experiment", log_dir=log_dir)
                logger.info(f"accelerator.device = {accelerator.device}")
                logger.info(f"seed = {settings.seed}")
                logger.info("Config:\n" + OmegaConf.to_yaml(cfg))
                reporter_kind = str(cfg.reporter).strip().lower()
                reporter_context = ReporterContext(
                    reporter_kind,
                    run_name=task.run_name(cfg),
                    config=cast(dict[str, Any], OmegaConf.to_container(cfg, resolve=True)),
                    mlflow=cast(dict[str, Any], OmegaConf.to_container(cfg.mlflow, resolve=True)),
                    wandb=cast(dict[str, Any], OmegaConf.to_container(cfg.wandb, resolve=True)),
                    resume_run_id=task.resume_run_id(cfg, reporter_kind),
                )
                reporter = stack.enter_context(reporter_context)
            except BaseException as exc:
                local_open_error = exc
        if _reporter_open_failed(local_open_error):
            _teardown_process_group(accelerator, logger, wait=False)
            if local_open_error is not None:
                raise local_open_error
            raise RuntimeError("Reporter failed to open on another rank")
        ctx = RunContext(cfg=cfg, accelerator=accelerator, logger=logger, reporter=reporter)
        try:
            run = task.run(ctx)
        except BaseException:
            _teardown_process_group(accelerator, logger, wait=False)
            raise
        _teardown_process_group(accelerator, logger, wait=True)
        raise_deferred_error()
        if accelerator.is_main_process:
            task.finish(run)
    return run

checkpoint_root(cfg, suffix='')

Return the directory holding the checkpoint_* folders of the training run.

Returns:

Type Description
str

"" when train.save_dir is empty or null, so saving and resume are disabled.

Parameters:

Name Type Description Default
cfg DictConfig

experiment config; train.save_dir, model, dataset, optimizer.lr, and train.batch_size select the directory.

required
suffix str

the training task's run_suffix.

''
Source code in src/mlkit/util/experiment/train.py
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
def checkpoint_root(cfg: DictConfig, suffix: str = "") -> str:
    """Return the directory holding the ``checkpoint_*`` folders of the training run.

    Returns:
        ``""`` when ``train.save_dir`` is empty or null, so saving and resume are disabled.

    Args:
        cfg: experiment config; ``train.save_dir``, model, dataset, ``optimizer.lr``,
            and ``train.batch_size`` select the directory.
        suffix: the training task's ``run_suffix``.
    """
    save_dir = cfg.train.get("save_dir")
    if save_dir is None or str(save_dir).strip() == "":
        return ""
    from mlkit.util.utils_for_main import get_new_save_dir

    return get_new_save_dir(save_dir, cfg, suffix=suffix)