Skip to content

report

Experiment-tracking backends and pipeline callbacks. For configuration and resume behavior, see Experiment tracking.

mlkit.util.report

NullReporter

Bases: Reporter

Reporter that discards metrics and does not open a run.

Source code in src/mlkit/util/report/base.py
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
class NullReporter(Reporter):
    """Reporter that discards metrics and does not open a run."""

    backend = "none"
    run_id: str | None = None

    def open(self) -> None:
        return None

    def close(self, *, success: bool) -> None:
        del success
        return None

    def log(self, data: Mapping[str, Any], *, step: int | None = None) -> None:
        del data, step

    def store_config(self, config: Mapping[str, Any]) -> None:
        del config

Reporter

Bases: ABC

Experiment-tracking backend (MLflow, wandb, or none).

Source code in src/mlkit/util/report/base.py
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
class Reporter(ABC):
    """Experiment-tracking backend (MLflow, wandb, or none)."""

    backend: str
    run_id: str | None

    @abstractmethod
    def open(self) -> None:
        """Open or attach to a tracking run."""

    @abstractmethod
    def close(self, *, success: bool) -> None:
        """Close the run when this reporter owns it."""

    @abstractmethod
    def log(self, data: Mapping[str, Any], *, step: int | None = None) -> None:
        """Record one metric payload."""

    @abstractmethod
    def store_config(self, config: Mapping[str, Any]) -> None:
        """Replace the stored config for the run."""

    def as_callback(self) -> ReportCallback:
        from .callback import ReportCallback

        return ReportCallback(self)

open() abstractmethod

Open or attach to a tracking run.

Source code in src/mlkit/util/report/base.py
19
20
21
@abstractmethod
def open(self) -> None:
    """Open or attach to a tracking run."""

close(*, success) abstractmethod

Close the run when this reporter owns it.

Source code in src/mlkit/util/report/base.py
23
24
25
@abstractmethod
def close(self, *, success: bool) -> None:
    """Close the run when this reporter owns it."""

log(data, *, step=None) abstractmethod

Record one metric payload.

Source code in src/mlkit/util/report/base.py
27
28
29
@abstractmethod
def log(self, data: Mapping[str, Any], *, step: int | None = None) -> None:
    """Record one metric payload."""

store_config(config) abstractmethod

Replace the stored config for the run.

Source code in src/mlkit/util/report/base.py
31
32
33
@abstractmethod
def store_config(self, config: Mapping[str, Any]) -> None:
    """Replace the stored config for the run."""

NullReportCallback

Bases: ReportCallback

Pipeline callback backed by :class:NullReporter.

Source code in src/mlkit/util/report/callback.py
57
58
59
60
61
class NullReportCallback(ReportCallback):
    """Pipeline callback backed by :class:`NullReporter`."""

    def __init__(self) -> None:
        super().__init__(NullReporter())

ReportCallback

Bases: BaseCallback

Forwards pipeline events to a :class:Reporter.

Source code in src/mlkit/util/report/callback.py
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
class ReportCallback(BaseCallback):
    """Forwards pipeline events to a :class:`Reporter`."""

    def __init__(self, reporter: Reporter) -> None:
        self._reporter = reporter

    @property
    def backend(self) -> str:
        return self._reporter.backend

    @backend.setter
    def backend(self, value: str) -> None:
        self._reporter.backend = value

    @property
    def run_id(self) -> str | None:
        return self._reporter.run_id

    @run_id.setter
    def run_id(self, value: str | None) -> None:
        self._reporter.run_id = value

    def on_event(self, event: CallbackEvent, *_args: object, **kwargs: object) -> None:
        if event is CallbackEvent.LOG:
            metrics = kwargs.get("metrics")
            if not isinstance(metrics, Mapping):
                return
            step = kwargs.get("step")
            self._reporter.log(cast(Mapping[str, Any], metrics), step=step if isinstance(step, int) else None)
            return
        if event is not CallbackEvent.PRE_FINALIZE_SAVE:
            return
        directory = kwargs.get("checkpoint_dir")
        if not isinstance(directory, str):
            return
        run_id = self._reporter.run_id
        if not isinstance(run_id, str) or run_id == "":
            return
        write_report_run(directory, backend=self._reporter.backend, run_id=run_id)

    def log(self, data: Mapping[str, Any], *, step: int | None = None) -> None:
        self._reporter.log(data, step=step)

    def store_config(self, config: Mapping[str, Any]) -> None:
        self._reporter.store_config(config)

ReporterContext

Class context manager: open on enter, close on exit.

Source code in src/mlkit/util/report/context.py
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
class ReporterContext:
    """Class context manager: ``open`` on enter, ``close`` on exit."""

    def __init__(
        self,
        kind: str,
        *,
        run_name: str,
        config: Mapping[str, Any],
        mlflow: Mapping[str, Any] | None = None,
        wandb: Mapping[str, Any] | None = None,
        resume_run_id: str | None = None,
    ) -> None:
        from . import get_reporter

        self._reporter = get_reporter(
            kind,
            run_name=run_name,
            config=config,
            mlflow=mlflow,
            wandb=wandb,
            resume_run_id=resume_run_id,
        )
        self._opened = False

    def __enter__(self) -> Reporter:
        self._reporter.open()
        self._opened = True
        return self._reporter

    def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
        del tb
        if self._opened:
            self._reporter.close(success=exc_type is None)
        return None

MLflowReporter

Bases: Reporter

MLflow tracking backend.

Source code in src/mlkit/util/report/mlflow.py
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
class MLflowReporter(Reporter):
    """MLflow tracking backend."""

    def __init__(
        self,
        *,
        tracking_uri: str | None,
        experiment: str,
        run_name: str,
        config: Mapping[str, Any],
        run_id: str | None = None,
        artifact_location: str | None = None,
    ) -> None:
        self.backend = "mlflow"
        self.tracking_uri = tracking_uri or None
        self.experiment = experiment
        self.run_name = run_name
        self.config = config
        self.run_id = run_id
        self.artifact_location = artifact_location or None
        self._auto_close = False
        self._opened = False

    def open(self) -> None:
        import mlflow

        if self.tracking_uri:
            active_uri = self.tracking_uri
        elif env_uri := os.environ.get("MLFLOW_TRACKING_URI"):
            active_uri = env_uri
        else:
            active_uri = "sqlite:///mlflow.db"
        _prepare_sqlite_dir(active_uri)
        if self.tracking_uri or not os.environ.get("MLFLOW_TRACKING_URI"):
            mlflow.set_tracking_uri(active_uri)
        if self.artifact_location and mlflow.get_experiment_by_name(self.experiment) is None:
            from mlflow.exceptions import MlflowException

            _prepare_artifact_dir(self.artifact_location)
            try:
                mlflow.create_experiment(self.experiment, artifact_location=self.artifact_location)
            except MlflowException as exc:
                if getattr(exc, "error_code", None) != "RESOURCE_ALREADY_EXISTS":
                    raise
        mlflow.set_experiment(self.experiment)

        started_new = False
        try:
            if self.run_id is not None:
                active = mlflow.start_run(run_id=self.run_id, run_name=self.run_name)
                self._auto_close = True
                started_new = True
            elif mlflow.active_run() is None:
                active = mlflow.start_run(run_name=self.run_name)
                self._auto_close = True
                started_new = True
            else:
                active = mlflow.active_run()
                self._auto_close = False

            if started_new and active is not None:
                run_id = active.info.run_id
                if not isinstance(run_id, str) or run_id == "":
                    raise RuntimeError("MLflow did not start a run")
                self.run_id = run_id
                mlflow.log_dict(_plain_data(self.config), "config.yaml")
            elif self.run_id is None and active is not None:
                attached_id = active.info.run_id
                if isinstance(attached_id, str) and attached_id != "":
                    self.run_id = attached_id
        except Exception:
            if started_new and self._auto_close:
                mlflow.end_run(status="FAILED")
            raise
        self._opened = True

    def close(self, *, success: bool) -> None:
        if not self._opened or not self._auto_close:
            return
        import mlflow

        mlflow.end_run(status="FINISHED" if success else "FAILED")
        self._opened = False

    def store_config(self, config: Mapping[str, Any]) -> None:
        self.config = config
        if not self._opened:
            return
        import mlflow

        mlflow.log_dict(_plain_data(self.config), "config.yaml")

    def log(self, data: Mapping[str, Any], *, step: int | None = None) -> None:
        metrics = _finite_metrics(data)
        if not metrics:
            return
        import mlflow

        if step is None:
            mlflow.log_metrics(metrics)
        else:
            mlflow.log_metrics(metrics, step=step)

WandbReporter

Bases: Reporter

wandb tracking backend.

Source code in src/mlkit/util/report/wandb.py
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
class WandbReporter(Reporter):
    """wandb tracking backend."""

    def __init__(
        self,
        *,
        project: str | None,
        name: str,
        group: str | None,
        entity: str | None,
        mode: str | None,
        reinit: bool,
        config: Mapping[str, Any],
        run_id: str | None = None,
    ) -> None:
        self.backend = "wandb"
        self.project = project
        self.name = name
        self.group = group
        self.entity = entity
        self.mode = mode
        self.reinit = reinit
        self.config = config
        self.run_id = run_id
        self._auto_close = False
        self._opened = False

    def open(self) -> None:
        import wandb

        if self.run_id is not None:
            run = wandb.init(
                reinit=self.reinit,
                mode=_wandb_mode(self.mode),
                project=self.project,
                name=self.name,
                group=self.group,
                entity=self.entity,
                config=_plain_data(self.config),
                id=self.run_id,
                resume="allow",
            )
            self._auto_close = True
        elif wandb.run is None:
            run = wandb.init(
                reinit=self.reinit,
                mode=_wandb_mode(self.mode),
                project=self.project,
                name=self.name,
                group=self.group,
                entity=self.entity,
                config=_plain_data(self.config),
            )
            self._auto_close = True
        else:
            run = wandb.run
            wandb.config.update(_plain_data(self.config), allow_val_change=True)
            self._auto_close = False

        try:
            run_id = None if run is None else run.id
            if not isinstance(run_id, str) or run_id == "":
                raise RuntimeError("wandb did not start a run")
            self.run_id = run_id
        except Exception:
            if self._auto_close:
                wandb.finish(exit_code=1)
            raise
        self._opened = True

    def close(self, *, success: bool) -> None:
        if not self._opened or not self._auto_close:
            return
        import wandb

        wandb.finish(exit_code=0 if success else 1)
        self._opened = False

    def store_config(self, config: Mapping[str, Any]) -> None:
        self.config = config
        if not self._opened:
            return
        import wandb

        wandb.config.update(_plain_data(self.config), allow_val_change=True)

    def log(self, data: Mapping[str, Any], *, step: int | None = None) -> None:
        import wandb

        payload = dict(data)
        if step is None:
            wandb.log(payload)
        else:
            wandb.log(payload, step=step)

resume_run_id(checkpoint_dir, backend)

Return the run id to continue, when the checkpoint was written by backend.

Parameters:

Name Type Description Default
checkpoint_dir str | None

latest checkpoint directory, or None when there is none.

required
backend str

reporter that is about to be opened.

required

Returns:

Type Description
str | None

The stored run id when the file exists and names the same backend.

Source code in src/mlkit/util/report/run_file.py
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
def resume_run_id(checkpoint_dir: str | None, backend: str) -> str | None:
    """Return the run id to continue, when the checkpoint was written by ``backend``.

    Args:
        checkpoint_dir: latest checkpoint directory, or ``None`` when there is none.
        backend: reporter that is about to be opened.

    Returns:
        The stored run id when the file exists and names the same backend.
    """
    if checkpoint_dir is None:
        return None
    found = read_report_run(checkpoint_dir)
    if found is None or found[0] != backend:
        return None
    return found[1]

write_report_run(directory, *, backend, run_id)

Store the backend run id inside a checkpoint directory.

Parameters:

Name Type Description Default
directory str

checkpoint directory.

required
backend str

reporter name, "mlflow" or "wandb".

required
run_id str

id of the run that was open when the checkpoint was saved.

required
Source code in src/mlkit/util/report/run_file.py
 7
 8
 9
10
11
12
13
14
15
16
17
def write_report_run(directory: str, *, backend: str, run_id: str) -> None:
    """Store the backend run id inside a checkpoint directory.

    Args:
        directory: checkpoint directory.
        backend: reporter name, ``"mlflow"`` or ``"wandb"``.
        run_id: id of the run that was open when the checkpoint was saved.
    """
    path = os.path.join(directory, REPORT_RUN_FILE)
    with open(path, "w", encoding="utf-8") as handle:
        json.dump({"backend": backend, "run_id": run_id}, handle)

get_reporter(kind, *, run_name, config, mlflow=None, wandb=None, resume_run_id=None)

Build a reporter for the given kind. The run is not open until open().

Source code in src/mlkit/util/report/__init__.py
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def get_reporter(
    kind: str,
    *,
    run_name: str,
    config: Mapping[str, Any],
    mlflow: Mapping[str, Any] | None = None,
    wandb: Mapping[str, Any] | None = None,
    resume_run_id: str | None = None,
) -> Reporter:
    """Build a reporter for the given kind. The run is not open until ``open()``."""
    kind_name = kind.strip().lower()
    if kind_name == "none":
        return NullReporter()
    if kind_name == "mlflow":
        settings = dict(mlflow or {})
        return MLflowReporter(
            tracking_uri=settings.get("tracking_uri") or None,
            experiment=str(settings.get("experiment") or "default"),
            run_name=run_name,
            config=config,
            run_id=resume_run_id,
            artifact_location=settings.get("artifact_location") or None,
        )
    if kind_name == "wandb":
        settings = dict(wandb or {})
        return WandbReporter(
            project=settings.get("project"),
            name=run_name,
            group=settings.get("group"),
            entity=settings.get("entity"),
            mode=settings.get("mode"),
            reinit=bool(settings.get("reinit", True)),
            config=config,
            run_id=resume_run_id,
        )
    raise ValueError(f"Unknown reporter: {kind}")