Skip to content

pipeline

Training-loop behavior is described in Training loop.

mlkit.pipeline

BaseCallback

Source code in src/mlkit/pipeline/callback.py
35
36
37
38
39
40
41
42
43
44
class BaseCallback(metaclass=ABCMeta):
    @abstractmethod
    def on_event(self, event: CallbackEvent, *args, **kwargs):
        """Handle one pipeline lifecycle event.

        Args:
            event: Callback event.
            *args: Positional payload for the event.
            **kwargs: Keyword payload for the event.
        """

on_event(event, *args, **kwargs) abstractmethod

Handle one pipeline lifecycle event.

Parameters:

Name Type Description Default
event CallbackEvent

Callback event.

required
*args

Positional payload for the event.

()
**kwargs

Keyword payload for the event.

{}
Source code in src/mlkit/pipeline/callback.py
36
37
38
39
40
41
42
43
44
@abstractmethod
def on_event(self, event: CallbackEvent, *args, **kwargs):
    """Handle one pipeline lifecycle event.

    Args:
        event: Callback event.
        *args: Positional payload for the event.
        **kwargs: Keyword payload for the event.
    """

CallbackManager

Source code in src/mlkit/pipeline/callback.py
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
class CallbackManager:
    def __init__(self):
        self.callbacks: List[BaseCallback] = []

    def add_callback(self, callback: BaseCallback):
        """Append one callback.

        Args:
            callback: Callback instance to register.
        """
        if callback is not None:
            self.callbacks.append(callback)

    def add_callbacks(self, callbacks: Optional[List[BaseCallback]]):
        """Append multiple callbacks.

        Args:
            callbacks: Callbacks to register.
        """
        if callbacks is not None and len(callbacks) > 0:
            self.callbacks.extend(callbacks)

    def trigger(self, event: CallbackEvent, *args, **kwargs):
        """Invoke every registered callback for ``event``.

        Args:
            event: Callback event.
            *args: Positional payload.
            **kwargs: Keyword payload.
        """
        for callback in self.callbacks:
            callback.on_event(event, *args, **kwargs)

add_callback(callback)

Append one callback.

Parameters:

Name Type Description Default
callback BaseCallback

Callback instance to register.

required
Source code in src/mlkit/pipeline/callback.py
51
52
53
54
55
56
57
58
def add_callback(self, callback: BaseCallback):
    """Append one callback.

    Args:
        callback: Callback instance to register.
    """
    if callback is not None:
        self.callbacks.append(callback)

add_callbacks(callbacks)

Append multiple callbacks.

Parameters:

Name Type Description Default
callbacks Optional[List[BaseCallback]]

Callbacks to register.

required
Source code in src/mlkit/pipeline/callback.py
60
61
62
63
64
65
66
67
def add_callbacks(self, callbacks: Optional[List[BaseCallback]]):
    """Append multiple callbacks.

    Args:
        callbacks: Callbacks to register.
    """
    if callbacks is not None and len(callbacks) > 0:
        self.callbacks.extend(callbacks)

trigger(event, *args, **kwargs)

Invoke every registered callback for event.

Parameters:

Name Type Description Default
event CallbackEvent

Callback event.

required
*args

Positional payload.

()
**kwargs

Keyword payload.

{}
Source code in src/mlkit/pipeline/callback.py
69
70
71
72
73
74
75
76
77
78
def trigger(self, event: CallbackEvent, *args, **kwargs):
    """Invoke every registered callback for ``event``.

    Args:
        event: Callback event.
        *args: Positional payload.
        **kwargs: Keyword payload.
    """
    for callback in self.callbacks:
        callback.on_event(event, *args, **kwargs)

DistributedPipeline

Bases: BasePipeline

Source code in src/mlkit/pipeline/distributed_pipeline.py
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
@inherit_docstrings
class DistributedPipeline(BasePipeline):
    def __init__(
        self,
        model: torch.nn.Module,
        dataset: Union[torch.utils.data.Dataset, datasets.Dataset],
        optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LRScheduler],
        training_config: DistributedTrainingConfig,
        log_config: LogConfig,
        logger: logging.Logger | None,
        collate_fn: Optional[Callable] = None,
        seed: int = 42,
        callbacks: Optional[List[BaseCallback]] = None,
        **kwargs,
    ):
        """Build an :class:`Accelerator`, prepare modules, and resume if configured.

        Args:
            seed: RNG seed applied before ``Accelerator`` construction.
        """
        accelerate.utils.set_seed(seed)
        self.accelerator = Accelerator(
            gradient_accumulation_steps=training_config.gradient_accumulation_steps,
            mixed_precision=training_config.mixed_precision,
            kwargs_handlers=[
                DistributedDataParallelKwargs(find_unused_parameters=training_config.find_unused_parameters),
            ],
        )

        super().__init__(
            model=model,
            dataset=dataset,
            optimizers=optimizers,
            training_config=training_config,
            log_config=log_config,
            collate_fn=collate_fn,
            logger=logger,
            callbacks=callbacks,
            load_checkpoint=False,
            **kwargs,
        )

        # Prepare everything for distributed training BEFORE loading
        self.model, self.optimizer, self.dataloader, self.scheduler = self.accelerator.prepare(
            self.model, self.optimizer, self.dataloader, self.scheduler
        )

        # Load checkpoint AFTER prepare to ensure proper state restoration
        if self.training_config.save_dir is not None and self.training_config.save_dir != "":
            self.load()
            self.training_config = training_config

        if self.accelerator.is_main_process:
            assert self.logger is not None, f"Error from {self.__class__.__name__}: logger is required"
            self.logger.info("Using distributed training with accelerate")
            self.logger.info(f"Number of processes: {self.accelerator.num_processes}")
            self.logger.info(f"Current device: {self.accelerator.device}")

    def gradient_clip(self) -> None:
        model = self.model
        if self.training_config.grad_clip_strategy == "norm":
            self.accelerator.clip_grad_norm_(
                model.parameters(),
                max_norm=self.training_config.max_grad_norm,
                norm_type=2,
            )
        if self.training_config.grad_clip_strategy == "value":
            self.accelerator.clip_grad_value_(
                model.parameters(),
                clip_value=self.training_config.max_grad_value,
            )

    def train_a_step(self, batch: Dict[str, Any]):
        self.trigger_callbacks(event=CallbackEvent.STEP_START, batch=batch)
        model: torch.nn.Module = self.model
        optimizer = self.optimizer
        scheduler = self.scheduler
        model.train()

        result = {}

        should_sync = (
            self.training_state.current_global_step + 1
        ) % self.training_config.gradient_accumulation_steps == 0
        ctx = self.accelerator.no_sync(model=model) if not should_sync else nullcontext()
        with ctx:
            raw_loss = self.compute_loss(model, batch)
            if isinstance(raw_loss, torch.Tensor):
                loss = raw_loss
            else:
                loss = cast(torch.Tensor, cast(Dict[str, Any], raw_loss)["loss"])
            self.trigger_callbacks(event=CallbackEvent.PRE_BACKWARD)
            self.accelerator.backward(loss)
            self.trigger_callbacks(event=CallbackEvent.POST_BACKWARD)
        if should_sync:
            self.gradient_clip()
            self.trigger_callbacks(event=CallbackEvent.PRE_OPTIMIZER_STEP)
            optimizer.step()
            self.trigger_callbacks(event=CallbackEvent.POST_OPTIMIZER_STEP)
            optimizer.zero_grad()

        if scheduler is not None:
            scheduler.step()

        result["loss"] = loss.item()
        result["lr"] = scheduler.get_last_lr()[0]
        result["global_step"] = self.training_state.current_global_step
        self._include_model_metrics(result, raw_loss)

        # A backend failure is remembered. Every rank hears about it on a logging step.
        if self._log_interval_due("steps"):
            self.log(result, step=self.training_state.current_global_step)
            self._sync_halt()

        self.trigger_callbacks(event=CallbackEvent.STEP_END, batch=batch)
        return result

    def _share_flag(self, local: bool) -> bool:
        """Share one rank-local boolean with every rank.

        ``num_processes == 1`` returns ``local``. Otherwise the rank where
        ``local`` is true sets Accelerate's trigger, and every rank checks it.
        The check consumes the trigger. Every rank must enter this call
        together, and the flag must be consumed before the next phase sets it.

        Args:
            local: this rank's value.

        Returns:
            True when any rank passed true.
        """
        if self.accelerator.num_processes == 1:
            return local
        if local:
            self.accelerator.set_trigger()
        return bool(self.accelerator.check_trigger())

    def _halt_if_any_rank_failed(self) -> bool:
        """Share a deferred error stored on the main process.

        ``num_processes == 1`` reads the local error. Otherwise only the main
        process sets the trigger, and every rank checks it.
        """
        if self.accelerator.num_processes == 1:
            return has_deferred_error()
        return self._share_flag(self.accelerator.is_main_process and has_deferred_error())

    def _abort_uncommitted_save(self, temp_dir: str, error: Exception | None) -> None:
        """Drop an uncommitted checkpoint and stop every rank.

        The rank that caught ``error`` remembers that exception. A failure on
        another rank leaves this process with ``error is None``. The main
        process still remembers a peer failure, because the report callback and
        ``post_train`` only see the main process. The published checkpoint
        directory is left untouched.

        Args:
            temp_dir: temporary directory for this attempt.
            error: exception caught on this rank, or ``None`` when another rank
                failed.
        """
        if self.accelerator.is_main_process:
            if self.logger is not None:
                if error is not None:
                    self.logger.error(f"Failed to save checkpoint: {error}")
                else:
                    self.logger.error("Failed to save checkpoint on another rank")
            shutil.rmtree(temp_dir, ignore_errors=True)
        if error is not None:
            defer_error(error)
        elif self.accelerator.is_main_process:
            defer_error(RuntimeError("Checkpoint save failed on another rank"))
        self._halt = True

    def save(self) -> None:
        self.trigger_callbacks(event=CallbackEvent.PRE_SAVE)
        save_dir = self.training_config.save_dir
        if save_dir is None or save_dir == "":
            return

        epoch = self.training_state.current_epoch
        step = self.training_state.current_step_in_epoch
        global_step = self.training_state.current_global_step
        checkpoint_name = self._get_checkpoint_name(epoch, step, global_step)
        temp_checkpoint_dir = os.path.join(save_dir, f"tmp_{checkpoint_name}")
        final_checkpoint_dir = os.path.join(save_dir, checkpoint_name)

        # The global main decides whether this checkpoint is already published.
        already_saved = self.accelerator.is_main_process and os.path.exists(final_checkpoint_dir)
        if self._share_flag(already_saved):
            return

        prepare_error: Exception | None = None
        if self.accelerator.is_main_process:
            try:
                os.makedirs(save_dir, exist_ok=True)
                os.makedirs(temp_checkpoint_dir, exist_ok=True)
            except Exception as exc:
                prepare_error = exc
        if self._share_flag(prepare_error is not None):
            self._abort_uncommitted_save(temp_checkpoint_dir, prepare_error)
            return

        # save_state uses collectives. A failure inside that collective can
        # still leave the other ranks there; this joins ranks that have returned.
        state_error: Exception | None = None
        try:
            self.accelerator.save_state(temp_checkpoint_dir)
        except Exception as exc:
            state_error = exc
        if self._share_flag(state_error is not None):
            self._abort_uncommitted_save(temp_checkpoint_dir, state_error)
            return

        # Other ranks are done writing. Metadata and rename are one publish.
        publish_error: Exception | None = None
        if self.accelerator.is_main_process:
            try:
                for base_name in [
                    "training_state",
                    "training_config",
                    "log_config",
                ]:
                    file_path = os.path.join(temp_checkpoint_dir, f"{base_name}.pth")
                    torch.save(getattr(self, base_name), file_path)
                self.trigger_callbacks(event=CallbackEvent.PRE_FINALIZE_SAVE, checkpoint_dir=temp_checkpoint_dir)
                os.rename(temp_checkpoint_dir, final_checkpoint_dir)
                if self.logger is not None:
                    self.logger.info(f"Model saved to {final_checkpoint_dir}")
            except Exception as exc:
                publish_error = exc
        if self._share_flag(publish_error is not None):
            self._abort_uncommitted_save(temp_checkpoint_dir, publish_error)
            return

        if self.accelerator.is_main_process:
            try:
                self._cleanup_old_checkpoints(save_dir=save_dir)
            except Exception as exc:
                if self.logger is not None:
                    self.logger.error(f"Failed to delete old checkpoints: {exc}")
        if self._halt_if_any_rank_failed():
            self._halt = True
        self.trigger_callbacks(event=CallbackEvent.POST_SAVE)

    def load(self) -> None:
        self.trigger_callbacks(event=CallbackEvent.PRE_LOAD)
        # check load condition ============================================================================
        checkpoint_dir = self.get_latest_checkpoint_dir()
        if checkpoint_dir is None or len(os.listdir(checkpoint_dir)) <= 0:
            return

        # load ============================================================================================
        self.accelerator.load_state(checkpoint_dir, load_kwargs={"weights_only": False})

        # Load training metadata
        for base_name in ["training_state", "training_config", "log_config"]:
            file_path = os.path.join(checkpoint_dir, f"{base_name}.pth")
            if not os.path.exists(file_path):
                if self.accelerator.is_main_process and self.logger is not None:
                    self.logger.info(f"File {file_path} does not exist")
                continue
            setattr(self, base_name, torch.load(file_path, weights_only=False))

        if self.accelerator.is_main_process and self.logger is not None:
            self.logger.info(f"Model loaded from {checkpoint_dir}")
        self.trigger_callbacks(event=CallbackEvent.POST_LOAD)

    def trigger_callbacks(self, event: CallbackEvent, **kwargs):
        """Trigger all callbacks for a given event

        Args:
            event: the event to trigger
            **kwargs: the keyword arguments to pass to the callback
        """
        super().trigger_callbacks(
            event=event,
            accelerator=self.accelerator,
            **kwargs,
        )

__init__(model, dataset, optimizers, training_config, log_config, logger, collate_fn=None, seed=42, callbacks=None, **kwargs)

Build an :class:Accelerator, prepare modules, and resume if configured.

Parameters:

Name Type Description Default
seed int

RNG seed applied before Accelerator construction.

42
Source code in src/mlkit/pipeline/distributed_pipeline.py
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
def __init__(
    self,
    model: torch.nn.Module,
    dataset: Union[torch.utils.data.Dataset, datasets.Dataset],
    optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LRScheduler],
    training_config: DistributedTrainingConfig,
    log_config: LogConfig,
    logger: logging.Logger | None,
    collate_fn: Optional[Callable] = None,
    seed: int = 42,
    callbacks: Optional[List[BaseCallback]] = None,
    **kwargs,
):
    """Build an :class:`Accelerator`, prepare modules, and resume if configured.

    Args:
        seed: RNG seed applied before ``Accelerator`` construction.
    """
    accelerate.utils.set_seed(seed)
    self.accelerator = Accelerator(
        gradient_accumulation_steps=training_config.gradient_accumulation_steps,
        mixed_precision=training_config.mixed_precision,
        kwargs_handlers=[
            DistributedDataParallelKwargs(find_unused_parameters=training_config.find_unused_parameters),
        ],
    )

    super().__init__(
        model=model,
        dataset=dataset,
        optimizers=optimizers,
        training_config=training_config,
        log_config=log_config,
        collate_fn=collate_fn,
        logger=logger,
        callbacks=callbacks,
        load_checkpoint=False,
        **kwargs,
    )

    # Prepare everything for distributed training BEFORE loading
    self.model, self.optimizer, self.dataloader, self.scheduler = self.accelerator.prepare(
        self.model, self.optimizer, self.dataloader, self.scheduler
    )

    # Load checkpoint AFTER prepare to ensure proper state restoration
    if self.training_config.save_dir is not None and self.training_config.save_dir != "":
        self.load()
        self.training_config = training_config

    if self.accelerator.is_main_process:
        assert self.logger is not None, f"Error from {self.__class__.__name__}: logger is required"
        self.logger.info("Using distributed training with accelerate")
        self.logger.info(f"Number of processes: {self.accelerator.num_processes}")
        self.logger.info(f"Current device: {self.accelerator.device}")

trigger_callbacks(event, **kwargs)

Trigger all callbacks for a given event

Parameters:

Name Type Description Default
event CallbackEvent

the event to trigger

required
**kwargs

the keyword arguments to pass to the callback

{}
Source code in src/mlkit/pipeline/distributed_pipeline.py
341
342
343
344
345
346
347
348
349
350
351
352
def trigger_callbacks(self, event: CallbackEvent, **kwargs):
    """Trigger all callbacks for a given event

    Args:
        event: the event to trigger
        **kwargs: the keyword arguments to pass to the callback
    """
    super().trigger_callbacks(
        event=event,
        accelerator=self.accelerator,
        **kwargs,
    )

DistributedTrainingConfig

Bases: TrainingConfig

Source code in src/mlkit/pipeline/distributed_pipeline.py
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
@inherit_docstrings
class DistributedTrainingConfig(TrainingConfig):
    def __init__(
        self,
        n_epochs: int = 100,
        batch_size: int = 4,
        device: str = "cuda",
        save_strategy: Literal["epochs", "steps", None] = "epochs",
        save_dir: str | None = None,
        save_steps: int = 10,
        save_epochs: int = 1,
        save_total_limit: int = 5,
        num_workers: int = 4,
        train_shuffle: bool = True,
        eval_strategy: Literal["epochs", "steps"] | None = None,
        eval_steps: int = 500,
        eval_epochs: int = 1,
        grad_clip_strategy: Literal["norm", "value", None] = "norm",
        max_grad_norm: float = 1.0,
        max_grad_value: float = 1.0,
        gradient_accumulation_steps: int = 1,
        mixed_precision: str = "fp16",
        find_unused_parameters: bool = False,
        **kwargs,
    ):
        """Accelerate-specific training settings (mixed precision, DDP flags).

        Shared schedule fields are documented on :class:`TrainingConfig`.
        """

        super().__init__(
            n_epochs=n_epochs,
            batch_size=batch_size,
            device=device,
            save_strategy=save_strategy,
            save_dir=save_dir,
            save_steps=save_steps,
            save_epochs=save_epochs,
            save_total_limit=save_total_limit,
            num_workers=num_workers,
            train_shuffle=train_shuffle,
            eval_strategy=eval_strategy,
            eval_steps=eval_steps,
            eval_epochs=eval_epochs,
            grad_clip_strategy=grad_clip_strategy,
            max_grad_norm=max_grad_norm,
            max_grad_value=max_grad_value,
            gradient_accumulation_steps=gradient_accumulation_steps,
            **kwargs,
        )
        self.mixed_precision = mixed_precision
        self.find_unused_parameters = find_unused_parameters

__init__(n_epochs=100, batch_size=4, device='cuda', save_strategy='epochs', save_dir=None, save_steps=10, save_epochs=1, save_total_limit=5, num_workers=4, train_shuffle=True, eval_strategy=None, eval_steps=500, eval_epochs=1, grad_clip_strategy='norm', max_grad_norm=1.0, max_grad_value=1.0, gradient_accumulation_steps=1, mixed_precision='fp16', find_unused_parameters=False, **kwargs)

Accelerate-specific training settings (mixed precision, DDP flags).

Shared schedule fields are documented on :class:TrainingConfig.

Source code in src/mlkit/pipeline/distributed_pipeline.py
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def __init__(
    self,
    n_epochs: int = 100,
    batch_size: int = 4,
    device: str = "cuda",
    save_strategy: Literal["epochs", "steps", None] = "epochs",
    save_dir: str | None = None,
    save_steps: int = 10,
    save_epochs: int = 1,
    save_total_limit: int = 5,
    num_workers: int = 4,
    train_shuffle: bool = True,
    eval_strategy: Literal["epochs", "steps"] | None = None,
    eval_steps: int = 500,
    eval_epochs: int = 1,
    grad_clip_strategy: Literal["norm", "value", None] = "norm",
    max_grad_norm: float = 1.0,
    max_grad_value: float = 1.0,
    gradient_accumulation_steps: int = 1,
    mixed_precision: str = "fp16",
    find_unused_parameters: bool = False,
    **kwargs,
):
    """Accelerate-specific training settings (mixed precision, DDP flags).

    Shared schedule fields are documented on :class:`TrainingConfig`.
    """

    super().__init__(
        n_epochs=n_epochs,
        batch_size=batch_size,
        device=device,
        save_strategy=save_strategy,
        save_dir=save_dir,
        save_steps=save_steps,
        save_epochs=save_epochs,
        save_total_limit=save_total_limit,
        num_workers=num_workers,
        train_shuffle=train_shuffle,
        eval_strategy=eval_strategy,
        eval_steps=eval_steps,
        eval_epochs=eval_epochs,
        grad_clip_strategy=grad_clip_strategy,
        max_grad_norm=max_grad_norm,
        max_grad_value=max_grad_value,
        gradient_accumulation_steps=gradient_accumulation_steps,
        **kwargs,
    )
    self.mixed_precision = mixed_precision
    self.find_unused_parameters = find_unused_parameters

BasePipeline

Source code in src/mlkit/pipeline/pipeline.py
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
class BasePipeline(metaclass=ABCMeta):
    def __init__(
        self,
        model: torch.nn.Module,
        dataset: Union[torch.utils.data.Dataset, datasets.Dataset],
        optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LRScheduler],
        training_config: TrainingConfig,
        log_config: LogConfig,
        logger: logging.Logger | None,
        collate_fn: Optional[Callable] = None,
        callbacks: Optional[List[BaseCallback]] = None,
        load_checkpoint: bool = True,
        *_args,
        **_kwargs,
    ):
        """Wire model, data, optimizers, configs, callbacks, and optional checkpoint resume."""
        super().__init__()
        # callbacks
        self.callback_manager = CallbackManager()
        self.callback_manager.add_callbacks(callbacks=callbacks or [])

        # model, dataset, optimizers, log, training,
        self.model = model
        self.dataset = dataset
        self.optimizer, self.scheduler = optimizers
        self.training_config = training_config
        self.log_config = log_config
        self.training_state = TrainingState()
        self._halt = False
        self.logger = logger.getChild(type(self).__name__) if logger is not None else None

        self.dataloader = torch.utils.data.DataLoader(
            cast(torch.utils.data.Dataset, self.dataset),
            batch_size=self.training_config.batch_size,
            shuffle=self.training_config.train_shuffle,
            collate_fn=collate_fn,
            num_workers=self.training_config.num_workers,
        )

        if load_checkpoint and self.training_config.save_dir is not None and self.training_config.save_dir != "":
            self.load()
            self.training_config = training_config

    @abstractmethod
    def compute_loss(self, model: torch.nn.Module, batch: dict) -> Tensor | dict:
        """Compute loss for one batch."""

    def train(self) -> None:
        """Run the epoch-based training loop until ``n_epochs`` or halt."""
        self.trigger_callbacks(event=CallbackEvent.TRAINING_START)
        n_epochs = self.training_config.n_epochs

        for epoch in tqdm(
            range(n_epochs),
            desc="training",
            mininterval=0,
            disable=not self._is_local_main_process(),
        ):
            if epoch < self.training_state.current_epoch:
                continue
            completed = self.train_an_epoch()
            if completed:
                self.training_state.current_epoch += 1

            if self._halt:
                break
            if self._can_eval(flag="epochs"):
                self.eval()
            if self._halt:
                break
            if self._can_save(flag="epochs"):
                self.save()
            if self._halt:
                break
        self.save()
        self._sync_halt()
        self.trigger_callbacks(event=CallbackEvent.TRAINING_END)

    def train_an_epoch(self) -> bool:
        """Train one epoch.

        Returns:
            True when every step in the epoch ran. A halt after that still
            counts as complete. A halt inside the step loop does not.
        """
        self.trigger_callbacks(event=CallbackEvent.EPOCH_START)
        resume_step = self.training_state.current_step_in_epoch
        active_dataloader = skip_first_batches(self.dataloader, resume_step) if resume_step > 0 else self.dataloader
        result = None
        completed = True
        with tqdm(
            active_dataloader,
            desc=f"epoch {self.training_state.current_epoch}",
            total=len(self.dataloader),
            initial=resume_step,
            position=1,
            leave=False,
            mininterval=0,
            disable=not self._is_local_main_process(),
        ) as pbar:
            for batch in pbar:
                result = self.train_a_step(batch)
                self.training_state.current_step_in_epoch += 1
                self.training_state.current_global_step += 1

                if self._halt:
                    completed = False
                    break
                if self._can_eval(flag="steps"):
                    self.eval()
                if self._halt:
                    completed = False
                    break
                if self._can_save(flag="steps"):
                    self.save()
                if self._halt:
                    completed = False
                    break
                pbar.set_postfix(loss=result["loss"])

        if completed and result is not None and self._log_interval_due("epochs"):
            self.log(result, step=self.training_state.current_epoch)
        if self._log_interval_due("epochs"):
            self._sync_halt()
        if completed:
            self.training_state.current_step_in_epoch = 0
        self.trigger_callbacks(event=CallbackEvent.EPOCH_END)
        return completed

    def train_a_step(self, batch: Dict[str, Any]) -> dict:
        """Run forward, backward, optional clip/step, and logging for one batch.

        Args:
            batch: Training batch moved to ``training_config.device``.
        """
        self.trigger_callbacks(event=CallbackEvent.STEP_START, batch=batch)
        model = self.model
        optimizer = self.optimizer
        scheduler = self.scheduler
        device = self.training_config.device
        model.train()
        for key, value in batch.items():
            if type(value) is torch.Tensor:
                batch[key] = value.to(device)
        model = model.to(device)

        raw_loss = self.compute_loss(model, batch)
        if isinstance(raw_loss, Tensor):
            loss = raw_loss
        else:
            loss = cast(Tensor, cast(Dict[str, Any], raw_loss)["loss"])
        loss = loss / self.training_config.gradient_accumulation_steps
        self.trigger_callbacks(event=CallbackEvent.PRE_BACKWARD, loss=loss)
        loss.backward()
        self.trigger_callbacks(event=CallbackEvent.POST_BACKWARD, loss=loss)

        if ((self.training_state.current_global_step + 1) % self.training_config.gradient_accumulation_steps) == 0:
            self.gradient_clip()
            self.trigger_callbacks(event=CallbackEvent.PRE_OPTIMIZER_STEP)
            optimizer.step()
            self.trigger_callbacks(event=CallbackEvent.POST_OPTIMIZER_STEP)
            optimizer.zero_grad()

        if scheduler is not None:
            scheduler.step()

        result = {"loss": loss.item()}
        self._include_model_metrics(result, raw_loss)
        if self._log_interval_due("steps"):
            self.log(result, step=self.training_state.current_global_step)
        self.trigger_callbacks(event=CallbackEvent.STEP_END, batch=batch)
        return result

    def gradient_clip(self) -> None:
        """Clip gradients when ``grad_clip_strategy`` is ``norm`` or ``value``."""
        model = self.model
        if self.training_config.grad_clip_strategy == "norm":
            torch.nn.utils.clip_grad_norm_(
                model.parameters(),
                max_norm=self.training_config.max_grad_norm,
                norm_type=2,
                error_if_nonfinite=False,
            )
        if self.training_config.grad_clip_strategy == "value":
            torch.nn.utils.clip_grad_value_(
                model.parameters(),
                clip_value=self.training_config.max_grad_value,
            )

    def eval(self):
        """Evaluate the model; subclasses implement distributed gathering."""

    def _halt_if_any_rank_failed(self) -> bool:
        """Return whether a rank-local failure should stop this training loop.

        A single-process pipeline stops when this process remembered a failure.
        DistributedPipeline shares that fact across ranks before returning.
        """
        return has_deferred_error()

    def _sync_halt(self) -> None:
        """Stop this rank when a shared or local failure is waiting."""
        if self._halt_if_any_rank_failed():
            self._halt = True

    @property
    def halted(self) -> bool:
        """Whether a shared or local failure stopped training on this rank."""
        return self._halt

    def _is_global_main_process(self) -> bool:
        """Return whether this process records metrics.

        A pipeline without an accelerator is the only process.
        """
        accelerator = getattr(self, "accelerator", None)
        return True if accelerator is None else bool(accelerator.is_main_process)

    def log(self, metrics: Mapping[str, Any], *, step: int | None = None) -> None:
        """Record one metric payload on the global main process.

        Callbacks receive a ``LOG`` event. A callback failure is remembered, and
        the text line is still written when a logger is attached.

        Args:
            metrics: values to record.
            step: step index for this payload.
        """
        if not self._is_global_main_process():
            return
        try:
            self.trigger_callbacks(event=CallbackEvent.LOG, metrics=metrics, step=step)
        except Exception as exc:
            defer_error(exc)
            if self.logger is not None:
                self.logger.error("log failed: %s", exc)
        if self.logger is not None:
            self.logger.info(_format_log(metrics, step))

    def _include_model_metrics(self, result: Dict[str, Any], raw_loss: object) -> None:
        """Copy ``metrics`` from a model output into the step result.

        Args:
            result: payload that will be logged for this step.
            raw_loss: value returned by ``compute_loss``.
        """
        if not isinstance(raw_loss, dict):
            return
        payload = cast(dict[str, Any], raw_loss)
        model_output = payload.get("model_output")
        source = cast(dict[str, Any], model_output) if isinstance(model_output, dict) else payload
        extra = source.get("metrics")
        if isinstance(extra, dict):
            result.update(cast(dict[str, Any], extra))

    def _is_local_main_process(self) -> bool:
        """Return True if this process should emit UI output (e.g. progress bars).

        Plain BasePipeline has no accelerator and is always the local main process.
        DistributedPipeline subclasses expose ``self.accelerator``.
        """
        accelerator = getattr(self, "accelerator", None)
        return True if accelerator is None else accelerator.is_local_main_process

    def _cleanup_old_checkpoints(self, save_dir):
        checkpoints = [d for d in os.listdir(save_dir) if d.startswith("checkpoint_")]

        def _checkpoint_sort_key(name: str) -> int:
            m = re.search(r"global(\d+)", name)
            return int(m.group(1)) if m is not None else 0

        checkpoints.sort(key=_checkpoint_sort_key)

        while len(checkpoints) > self.training_config.save_total_limit:
            oldest_checkpoint = checkpoints.pop(0)
            oldest_path = os.path.join(save_dir, oldest_checkpoint)
            shutil.rmtree(oldest_path)
            if self.logger is not None:
                self.logger.info(f"Deleted old checkpoint: {oldest_path}")

    def _get_checkpoint_name(self, epoch, step, global_step):
        return f"checkpoint_epoch{epoch}_step{step}_global{global_step}"

    def _can_save(self, flag: Literal["epochs", "steps"]):
        if (
            self.training_config.save_strategy != flag
            or self.training_config.save_dir is None
            or self.training_config.save_dir == ""
        ):
            return False
        if flag == "epochs":
            # current_epoch is already incremented after train_an_epoch() completes.
            return (
                self.training_state.current_epoch > 0
                and self.training_state.current_epoch % self.training_config.save_epochs == 0
            )
        if flag == "steps":
            step = self.training_state.current_global_step
            return step > 0 and step % self.training_config.save_steps == 0
        return False

    def _log_interval_due(self, flag: Literal["epochs", "steps"]) -> bool:
        """Return whether this step or epoch is a logging point on every rank.

        The logger is not consulted. Ranks agree because they share the log
        strategy and the training counters.

        Args:
            flag: ``"epochs"`` or ``"steps"``.

        Returns:
            True when the current counter matches the logging interval.
        """
        if self.log_config.log_strategy != flag or self.log_config.log_dir is None or self.log_config.log_dir == "":
            return False
        if flag == "epochs":
            return (self.training_state.current_epoch + 1) % self.log_config.log_epochs == 0
        if flag == "steps":
            return (self.training_state.current_global_step + 1) % self.log_config.log_steps == 0
        return False

    def _can_eval(self, flag: Literal["epochs", "steps"]):
        if self.training_config.eval_strategy != flag:
            return False
        if flag == "epochs":
            # current_epoch is already incremented after train_an_epoch() completes.
            return (
                self.training_state.current_epoch > 0
                and self.training_state.current_epoch % self.training_config.eval_epochs == 0
            )
        if flag == "steps":
            step = self.training_state.current_global_step
            return step > 0 and step % self.training_config.eval_steps == 0
        return False

    def save(self) -> None:
        """Write model, optimizer, and training state under ``save_dir``."""
        self.trigger_callbacks(event=CallbackEvent.PRE_SAVE)
        save_dir = self.training_config.save_dir
        if save_dir is None or save_dir == "":
            return
        os.makedirs(save_dir, exist_ok=True)

        epoch = self.training_state.current_epoch
        step = self.training_state.current_step_in_epoch
        global_step = self.training_state.current_global_step
        checkpoint_name = self._get_checkpoint_name(epoch, step, global_step)
        temp_checkpoint_dir = os.path.join(save_dir, f"tmp_{checkpoint_name}")
        final_checkpoint_dir = os.path.join(save_dir, checkpoint_name)
        if os.path.exists(final_checkpoint_dir):
            return

        os.makedirs(temp_checkpoint_dir, exist_ok=True)
        try:
            for base_name in [
                "model",
                "optimizer",
                "scheduler",
                "training_state",
                "training_config",
                "log_config",
            ]:
                file_path = os.path.join(temp_checkpoint_dir, f"{base_name}.pth")
                torch.save(getattr(self, base_name), file_path)
            self.trigger_callbacks(event=CallbackEvent.PRE_FINALIZE_SAVE, checkpoint_dir=temp_checkpoint_dir)
            os.rename(temp_checkpoint_dir, final_checkpoint_dir)
            if self.logger is not None:
                self.logger.info(f"Model saved to {final_checkpoint_dir}")
        except Exception as e:
            if self.logger is not None:
                self.logger.error(f"Failed to save checkpoint: {e}")
            shutil.rmtree(temp_checkpoint_dir, ignore_errors=True)
            raise
        try:
            self._cleanup_old_checkpoints(save_dir=save_dir)
        except Exception as exc:
            if self.logger is not None:
                self.logger.error(f"Failed to delete old checkpoints: {exc}")
        self.trigger_callbacks(event=CallbackEvent.POST_SAVE)

    def load(self) -> None:
        """Restore the latest checkpoint from ``save_dir`` when present."""
        self.trigger_callbacks(event=CallbackEvent.PRE_LOAD)
        # check load condition ============================================================================
        checkpoint_dir = self.get_latest_checkpoint_dir()
        if checkpoint_dir is None or len(os.listdir(checkpoint_dir)) <= 0:
            return

        # load ============================================================================================

        for base_name in [
            "model",
            "optimizer",
            "scheduler",
            "training_state",
            "training_config",
            "log_config",
        ]:
            file_path = os.path.join(checkpoint_dir, f"{base_name}.pth")
            if not os.path.exists(file_path):
                if self.logger is not None:
                    self.logger.info(f"File {file_path} does not exist")
                continue
            setattr(self, base_name, torch.load(file_path, weights_only=False))

        if self.logger is not None:
            self.logger.info(f"Model loaded from {checkpoint_dir}")
        self.trigger_callbacks(event=CallbackEvent.POST_LOAD)

    def get_latest_checkpoint_dir(self) -> str | None:
        return latest_checkpoint_dir(self.training_config.save_dir)

    def add_callbacks(self, callbacks: List[BaseCallback] | None):
        """Register extra pipeline callbacks.

        Args:
            callbacks: Callbacks to append, or ``None`` to skip.
        """
        self.callback_manager.add_callbacks(callbacks=callbacks)

    def trigger_callbacks(self, event: CallbackEvent, **kwargs):
        """Dispatch one callback event with training context and extra keywords.

        Args:
            event: Callback lifecycle event.
            **kwargs: Event-specific payload for callbacks.
        """
        self.callback_manager.trigger(
            event=event,
            training_config=self.training_config,
            log_config=self.log_config,
            training_state=self.training_state,
            logger=self.logger,
            model=self.model,
            dataloader=self.dataloader,
            optimizer=self.optimizer,
            scheduler=self.scheduler,
            **kwargs,
        )

halted property

Whether a shared or local failure stopped training on this rank.

__init__(model, dataset, optimizers, training_config, log_config, logger, collate_fn=None, callbacks=None, load_checkpoint=True, *_args, **_kwargs)

Wire model, data, optimizers, configs, callbacks, and optional checkpoint resume.

Source code in src/mlkit/pipeline/pipeline.py
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
def __init__(
    self,
    model: torch.nn.Module,
    dataset: Union[torch.utils.data.Dataset, datasets.Dataset],
    optimizers: Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LRScheduler],
    training_config: TrainingConfig,
    log_config: LogConfig,
    logger: logging.Logger | None,
    collate_fn: Optional[Callable] = None,
    callbacks: Optional[List[BaseCallback]] = None,
    load_checkpoint: bool = True,
    *_args,
    **_kwargs,
):
    """Wire model, data, optimizers, configs, callbacks, and optional checkpoint resume."""
    super().__init__()
    # callbacks
    self.callback_manager = CallbackManager()
    self.callback_manager.add_callbacks(callbacks=callbacks or [])

    # model, dataset, optimizers, log, training,
    self.model = model
    self.dataset = dataset
    self.optimizer, self.scheduler = optimizers
    self.training_config = training_config
    self.log_config = log_config
    self.training_state = TrainingState()
    self._halt = False
    self.logger = logger.getChild(type(self).__name__) if logger is not None else None

    self.dataloader = torch.utils.data.DataLoader(
        cast(torch.utils.data.Dataset, self.dataset),
        batch_size=self.training_config.batch_size,
        shuffle=self.training_config.train_shuffle,
        collate_fn=collate_fn,
        num_workers=self.training_config.num_workers,
    )

    if load_checkpoint and self.training_config.save_dir is not None and self.training_config.save_dir != "":
        self.load()
        self.training_config = training_config

compute_loss(model, batch) abstractmethod

Compute loss for one batch.

Source code in src/mlkit/pipeline/pipeline.py
169
170
171
@abstractmethod
def compute_loss(self, model: torch.nn.Module, batch: dict) -> Tensor | dict:
    """Compute loss for one batch."""

train()

Run the epoch-based training loop until n_epochs or halt.

Source code in src/mlkit/pipeline/pipeline.py
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def train(self) -> None:
    """Run the epoch-based training loop until ``n_epochs`` or halt."""
    self.trigger_callbacks(event=CallbackEvent.TRAINING_START)
    n_epochs = self.training_config.n_epochs

    for epoch in tqdm(
        range(n_epochs),
        desc="training",
        mininterval=0,
        disable=not self._is_local_main_process(),
    ):
        if epoch < self.training_state.current_epoch:
            continue
        completed = self.train_an_epoch()
        if completed:
            self.training_state.current_epoch += 1

        if self._halt:
            break
        if self._can_eval(flag="epochs"):
            self.eval()
        if self._halt:
            break
        if self._can_save(flag="epochs"):
            self.save()
        if self._halt:
            break
    self.save()
    self._sync_halt()
    self.trigger_callbacks(event=CallbackEvent.TRAINING_END)

train_an_epoch()

Train one epoch.

Returns:

Type Description
bool

True when every step in the epoch ran. A halt after that still

bool

counts as complete. A halt inside the step loop does not.

Source code in src/mlkit/pipeline/pipeline.py
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
def train_an_epoch(self) -> bool:
    """Train one epoch.

    Returns:
        True when every step in the epoch ran. A halt after that still
        counts as complete. A halt inside the step loop does not.
    """
    self.trigger_callbacks(event=CallbackEvent.EPOCH_START)
    resume_step = self.training_state.current_step_in_epoch
    active_dataloader = skip_first_batches(self.dataloader, resume_step) if resume_step > 0 else self.dataloader
    result = None
    completed = True
    with tqdm(
        active_dataloader,
        desc=f"epoch {self.training_state.current_epoch}",
        total=len(self.dataloader),
        initial=resume_step,
        position=1,
        leave=False,
        mininterval=0,
        disable=not self._is_local_main_process(),
    ) as pbar:
        for batch in pbar:
            result = self.train_a_step(batch)
            self.training_state.current_step_in_epoch += 1
            self.training_state.current_global_step += 1

            if self._halt:
                completed = False
                break
            if self._can_eval(flag="steps"):
                self.eval()
            if self._halt:
                completed = False
                break
            if self._can_save(flag="steps"):
                self.save()
            if self._halt:
                completed = False
                break
            pbar.set_postfix(loss=result["loss"])

    if completed and result is not None and self._log_interval_due("epochs"):
        self.log(result, step=self.training_state.current_epoch)
    if self._log_interval_due("epochs"):
        self._sync_halt()
    if completed:
        self.training_state.current_step_in_epoch = 0
    self.trigger_callbacks(event=CallbackEvent.EPOCH_END)
    return completed

train_a_step(batch)

Run forward, backward, optional clip/step, and logging for one batch.

Parameters:

Name Type Description Default
batch Dict[str, Any]

Training batch moved to training_config.device.

required
Source code in src/mlkit/pipeline/pipeline.py
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
def train_a_step(self, batch: Dict[str, Any]) -> dict:
    """Run forward, backward, optional clip/step, and logging for one batch.

    Args:
        batch: Training batch moved to ``training_config.device``.
    """
    self.trigger_callbacks(event=CallbackEvent.STEP_START, batch=batch)
    model = self.model
    optimizer = self.optimizer
    scheduler = self.scheduler
    device = self.training_config.device
    model.train()
    for key, value in batch.items():
        if type(value) is torch.Tensor:
            batch[key] = value.to(device)
    model = model.to(device)

    raw_loss = self.compute_loss(model, batch)
    if isinstance(raw_loss, Tensor):
        loss = raw_loss
    else:
        loss = cast(Tensor, cast(Dict[str, Any], raw_loss)["loss"])
    loss = loss / self.training_config.gradient_accumulation_steps
    self.trigger_callbacks(event=CallbackEvent.PRE_BACKWARD, loss=loss)
    loss.backward()
    self.trigger_callbacks(event=CallbackEvent.POST_BACKWARD, loss=loss)

    if ((self.training_state.current_global_step + 1) % self.training_config.gradient_accumulation_steps) == 0:
        self.gradient_clip()
        self.trigger_callbacks(event=CallbackEvent.PRE_OPTIMIZER_STEP)
        optimizer.step()
        self.trigger_callbacks(event=CallbackEvent.POST_OPTIMIZER_STEP)
        optimizer.zero_grad()

    if scheduler is not None:
        scheduler.step()

    result = {"loss": loss.item()}
    self._include_model_metrics(result, raw_loss)
    if self._log_interval_due("steps"):
        self.log(result, step=self.training_state.current_global_step)
    self.trigger_callbacks(event=CallbackEvent.STEP_END, batch=batch)
    return result

gradient_clip()

Clip gradients when grad_clip_strategy is norm or value.

Source code in src/mlkit/pipeline/pipeline.py
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
def gradient_clip(self) -> None:
    """Clip gradients when ``grad_clip_strategy`` is ``norm`` or ``value``."""
    model = self.model
    if self.training_config.grad_clip_strategy == "norm":
        torch.nn.utils.clip_grad_norm_(
            model.parameters(),
            max_norm=self.training_config.max_grad_norm,
            norm_type=2,
            error_if_nonfinite=False,
        )
    if self.training_config.grad_clip_strategy == "value":
        torch.nn.utils.clip_grad_value_(
            model.parameters(),
            clip_value=self.training_config.max_grad_value,
        )

eval()

Evaluate the model; subclasses implement distributed gathering.

Source code in src/mlkit/pipeline/pipeline.py
315
316
def eval(self):
    """Evaluate the model; subclasses implement distributed gathering."""

log(metrics, *, step=None)

Record one metric payload on the global main process.

Callbacks receive a LOG event. A callback failure is remembered, and the text line is still written when a logger is attached.

Parameters:

Name Type Description Default
metrics Mapping[str, Any]

values to record.

required
step int | None

step index for this payload.

None
Source code in src/mlkit/pipeline/pipeline.py
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
def log(self, metrics: Mapping[str, Any], *, step: int | None = None) -> None:
    """Record one metric payload on the global main process.

    Callbacks receive a ``LOG`` event. A callback failure is remembered, and
    the text line is still written when a logger is attached.

    Args:
        metrics: values to record.
        step: step index for this payload.
    """
    if not self._is_global_main_process():
        return
    try:
        self.trigger_callbacks(event=CallbackEvent.LOG, metrics=metrics, step=step)
    except Exception as exc:
        defer_error(exc)
        if self.logger is not None:
            self.logger.error("log failed: %s", exc)
    if self.logger is not None:
        self.logger.info(_format_log(metrics, step))

save()

Write model, optimizer, and training state under save_dir.

Source code in src/mlkit/pipeline/pipeline.py
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
def save(self) -> None:
    """Write model, optimizer, and training state under ``save_dir``."""
    self.trigger_callbacks(event=CallbackEvent.PRE_SAVE)
    save_dir = self.training_config.save_dir
    if save_dir is None or save_dir == "":
        return
    os.makedirs(save_dir, exist_ok=True)

    epoch = self.training_state.current_epoch
    step = self.training_state.current_step_in_epoch
    global_step = self.training_state.current_global_step
    checkpoint_name = self._get_checkpoint_name(epoch, step, global_step)
    temp_checkpoint_dir = os.path.join(save_dir, f"tmp_{checkpoint_name}")
    final_checkpoint_dir = os.path.join(save_dir, checkpoint_name)
    if os.path.exists(final_checkpoint_dir):
        return

    os.makedirs(temp_checkpoint_dir, exist_ok=True)
    try:
        for base_name in [
            "model",
            "optimizer",
            "scheduler",
            "training_state",
            "training_config",
            "log_config",
        ]:
            file_path = os.path.join(temp_checkpoint_dir, f"{base_name}.pth")
            torch.save(getattr(self, base_name), file_path)
        self.trigger_callbacks(event=CallbackEvent.PRE_FINALIZE_SAVE, checkpoint_dir=temp_checkpoint_dir)
        os.rename(temp_checkpoint_dir, final_checkpoint_dir)
        if self.logger is not None:
            self.logger.info(f"Model saved to {final_checkpoint_dir}")
    except Exception as e:
        if self.logger is not None:
            self.logger.error(f"Failed to save checkpoint: {e}")
        shutil.rmtree(temp_checkpoint_dir, ignore_errors=True)
        raise
    try:
        self._cleanup_old_checkpoints(save_dir=save_dir)
    except Exception as exc:
        if self.logger is not None:
            self.logger.error(f"Failed to delete old checkpoints: {exc}")
    self.trigger_callbacks(event=CallbackEvent.POST_SAVE)

load()

Restore the latest checkpoint from save_dir when present.

Source code in src/mlkit/pipeline/pipeline.py
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
def load(self) -> None:
    """Restore the latest checkpoint from ``save_dir`` when present."""
    self.trigger_callbacks(event=CallbackEvent.PRE_LOAD)
    # check load condition ============================================================================
    checkpoint_dir = self.get_latest_checkpoint_dir()
    if checkpoint_dir is None or len(os.listdir(checkpoint_dir)) <= 0:
        return

    # load ============================================================================================

    for base_name in [
        "model",
        "optimizer",
        "scheduler",
        "training_state",
        "training_config",
        "log_config",
    ]:
        file_path = os.path.join(checkpoint_dir, f"{base_name}.pth")
        if not os.path.exists(file_path):
            if self.logger is not None:
                self.logger.info(f"File {file_path} does not exist")
            continue
        setattr(self, base_name, torch.load(file_path, weights_only=False))

    if self.logger is not None:
        self.logger.info(f"Model loaded from {checkpoint_dir}")
    self.trigger_callbacks(event=CallbackEvent.POST_LOAD)

add_callbacks(callbacks)

Register extra pipeline callbacks.

Parameters:

Name Type Description Default
callbacks List[BaseCallback] | None

Callbacks to append, or None to skip.

required
Source code in src/mlkit/pipeline/pipeline.py
538
539
540
541
542
543
544
def add_callbacks(self, callbacks: List[BaseCallback] | None):
    """Register extra pipeline callbacks.

    Args:
        callbacks: Callbacks to append, or ``None`` to skip.
    """
    self.callback_manager.add_callbacks(callbacks=callbacks)

trigger_callbacks(event, **kwargs)

Dispatch one callback event with training context and extra keywords.

Parameters:

Name Type Description Default
event CallbackEvent

Callback lifecycle event.

required
**kwargs

Event-specific payload for callbacks.

{}
Source code in src/mlkit/pipeline/pipeline.py
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
def trigger_callbacks(self, event: CallbackEvent, **kwargs):
    """Dispatch one callback event with training context and extra keywords.

    Args:
        event: Callback lifecycle event.
        **kwargs: Event-specific payload for callbacks.
    """
    self.callback_manager.trigger(
        event=event,
        training_config=self.training_config,
        log_config=self.log_config,
        training_state=self.training_state,
        logger=self.logger,
        model=self.model,
        dataloader=self.dataloader,
        optimizer=self.optimizer,
        scheduler=self.scheduler,
        **kwargs,
    )

LogConfig

Source code in src/mlkit/pipeline/pipeline.py
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
class LogConfig:
    def __init__(
        self,
        log_dir: str = "logs",
        log_steps: int = 100,
        log_epochs: int = 1,
        log_strategy: Literal["epochs", "steps"] = "epochs",
        *_args,
        **_kwargs,
    ):
        """Directory and interval for file logging during training."""
        self.log_dir = log_dir
        self.log_steps = log_steps
        self.log_epochs = log_epochs
        self.log_strategy = log_strategy

__init__(log_dir='logs', log_steps=100, log_epochs=1, log_strategy='epochs', *_args, **_kwargs)

Directory and interval for file logging during training.

Source code in src/mlkit/pipeline/pipeline.py
74
75
76
77
78
79
80
81
82
83
84
85
86
87
def __init__(
    self,
    log_dir: str = "logs",
    log_steps: int = 100,
    log_epochs: int = 1,
    log_strategy: Literal["epochs", "steps"] = "epochs",
    *_args,
    **_kwargs,
):
    """Directory and interval for file logging during training."""
    self.log_dir = log_dir
    self.log_steps = log_steps
    self.log_epochs = log_epochs
    self.log_strategy = log_strategy

TrainingConfig

Source code in src/mlkit/pipeline/pipeline.py
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
class TrainingConfig:
    def __init__(
        self,
        n_epochs: int = 100,
        batch_size: int = 4,
        device: str = "cuda",
        save_strategy: Literal["epochs", "steps", None] = "epochs",
        save_dir: str | None = None,
        save_steps: int = 10,
        save_epochs: int = 1,
        save_total_limit: int = 5,
        num_workers: int = 4,
        train_shuffle: bool = True,
        eval_strategy: Literal["epochs", "steps"] | None = None,
        eval_steps: int = 500,
        eval_epochs: int = 1,
        grad_clip_strategy: Literal["norm", "value", None] = "norm",
        max_grad_norm: float = 1.0,
        max_grad_value: float = 1.0,
        gradient_accumulation_steps: int = 1,
        *_args,
        **_kwargs,
    ):
        """Training hyperparameters and checkpoint, eval, and save scheduling.

        See the Training loop guide (``docs/training.md``) for how each field drives
        the epoch and step clocks.
        """
        self.n_epochs = n_epochs
        self.batch_size = batch_size
        self.device = device
        self.save_strategy = save_strategy
        self.save_dir = save_dir
        self.save_steps = save_steps
        self.save_epochs = save_epochs
        self.num_workers = num_workers
        self.train_shuffle = train_shuffle
        self.save_total_limit = save_total_limit
        self.eval_strategy = eval_strategy
        self.eval_steps = eval_steps
        self.eval_epochs = eval_epochs
        self.grad_clip_strategy = grad_clip_strategy
        self.max_grad_norm = max_grad_norm
        self.max_grad_value = max_grad_value
        self.gradient_accumulation_steps = gradient_accumulation_steps

__init__(n_epochs=100, batch_size=4, device='cuda', save_strategy='epochs', save_dir=None, save_steps=10, save_epochs=1, save_total_limit=5, num_workers=4, train_shuffle=True, eval_strategy=None, eval_steps=500, eval_epochs=1, grad_clip_strategy='norm', max_grad_norm=1.0, max_grad_value=1.0, gradient_accumulation_steps=1, *_args, **_kwargs)

Training hyperparameters and checkpoint, eval, and save scheduling.

See the Training loop guide (docs/training.md) for how each field drives the epoch and step clocks.

Source code in src/mlkit/pipeline/pipeline.py
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def __init__(
    self,
    n_epochs: int = 100,
    batch_size: int = 4,
    device: str = "cuda",
    save_strategy: Literal["epochs", "steps", None] = "epochs",
    save_dir: str | None = None,
    save_steps: int = 10,
    save_epochs: int = 1,
    save_total_limit: int = 5,
    num_workers: int = 4,
    train_shuffle: bool = True,
    eval_strategy: Literal["epochs", "steps"] | None = None,
    eval_steps: int = 500,
    eval_epochs: int = 1,
    grad_clip_strategy: Literal["norm", "value", None] = "norm",
    max_grad_norm: float = 1.0,
    max_grad_value: float = 1.0,
    gradient_accumulation_steps: int = 1,
    *_args,
    **_kwargs,
):
    """Training hyperparameters and checkpoint, eval, and save scheduling.

    See the Training loop guide (``docs/training.md``) for how each field drives
    the epoch and step clocks.
    """
    self.n_epochs = n_epochs
    self.batch_size = batch_size
    self.device = device
    self.save_strategy = save_strategy
    self.save_dir = save_dir
    self.save_steps = save_steps
    self.save_epochs = save_epochs
    self.num_workers = num_workers
    self.train_shuffle = train_shuffle
    self.save_total_limit = save_total_limit
    self.eval_strategy = eval_strategy
    self.eval_steps = eval_steps
    self.eval_epochs = eval_epochs
    self.grad_clip_strategy = grad_clip_strategy
    self.max_grad_norm = max_grad_norm
    self.max_grad_value = max_grad_value
    self.gradient_accumulation_steps = gradient_accumulation_steps

TrainingState

Source code in src/mlkit/pipeline/pipeline.py
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
class TrainingState:
    def __init__(
        self,
        current_epoch: int = 0,
        current_step_in_epoch: int = 0,
        current_global_step: int = 0,
    ):
        """Epoch, in-epoch step, and global step counters for resume and intervals."""
        self.current_epoch = current_epoch
        self.current_step_in_epoch = current_step_in_epoch
        self.current_global_step = current_global_step

__init__(current_epoch=0, current_step_in_epoch=0, current_global_step=0)

Epoch, in-epoch step, and global step counters for resume and intervals.

Source code in src/mlkit/pipeline/pipeline.py
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def __init__(
    self,
    current_epoch: int = 0,
    current_step_in_epoch: int = 0,
    current_global_step: int = 0,
):
    """Epoch, in-epoch step, and global step counters for resume and intervals."""
    self.current_epoch = current_epoch
    self.current_step_in_epoch = current_step_in_epoch
    self.current_global_step = current_global_step