1import logging
2import os
3
4from omegaml.backends.basemodel import BaseModelBackend
5from omegaml.backends.tracking.base import TrackingProvider
6
7logger = logging.getLogger(__name__)
8
9
[docs]
10class ExperimentBackend(BaseModelBackend):
11 """ExperimentBackend provides storage of tracker configurations
12
13 Usage:
14
15 To log metrics and other data::
16
17 with om.runtime.experiment('myexp') as exp:
18 om.runtime.model('mymodel').fit(X, Y)
19 om.runtime.model('mymodel').score(X, Y) # automatically log score result
20 exp.log_metric('mymetric', value)
21 exp.log_param('myparam', value)
22 exp.log_artifact(X, 'X')
23 exp.log_artifact(Y, 'Y')
24 exp.log_artifact(om.models.metadata('mymodel'), 'mymodel')
25
26 To log data and automatically profile system data::
27
28 with om.runtime.experiment('myexp', provider='profiling') as exp:
29 om.runtime.model('mymodel').fit(X, Y)
30 om.runtime.model('mymodel').score(X, Y) # automatically log score result
31 exp.log_metric('mymetric', value)
32 exp.log_param('myparam', value)
33 exp.log_artifact(X, 'X')
34 exp.log_artifact(Y, 'Y')
35 exp.log_artifact(om.models.metadata('mymodel'), 'mymodel')
36
37 # profiling data contains metrics for cpu, memory and disk use
38 data = exp.data(event='profile')
39
40 To get back experiment data without running an experiment::
41
42 # recommended way
43 exp = om.runtime.experiment('myexp').use()
44 exp_df = exp.data()
45
46 # experiments exist in the models store
47 exp = om.models.get('experiments/myexp')
48 exp_df = exp.data()
49
50 See Also:
51
52 * :class:`omegaml.backends.tracking.OmegaSimpleTracker`
53 * :class:`omegaml.backends.tracking.OmegaProfilingTracker`
54 """
55
56 KIND = 'experiment.tracker'
57 exp_prefix = 'experiments/'
58
[docs]
59 @classmethod
60 def supports(self, obj, name, **kwargs):
61 return isinstance(obj, TrackingProvider)
62
[docs]
63 def put(self, obj, name, **kwargs):
64 name = f'{self.exp_prefix}{name}' if not name.startswith(self.exp_prefix) else name
65 # FIXME use proper pickle magic to avoid storing the store
66 store = obj._store
67 obj._store = None
68 obj._model_store = None
69 meta = super().put(obj, name, **kwargs)
70 meta.attributes.setdefault('tracking', {})
71 meta.attributes['tracking']['dataset'] = obj._data_name
72 meta.save()
73 obj._store = store
74 obj._model_store = self.model_store
75 return meta
76
[docs]
77 def get(self, name, raw=False, data_store=None, **kwargs):
78 data_store = data_store or self.data_store
79 assert data_store is not None, "experiments require a datastore, specify data_store=om.datasets"
80 tracker = super().get(name, **kwargs)
81 tracker._store = data_store
82 tracker._model_store = self.model_store
83 # fix for #452, maintain backwards compatibility
84 based_name = os.path.basename(name)
85 based_dataset = data_store.exists(f'.{self.exp_prefix}{based_name}')
86 actual_name = name.replace(self.exp_prefix, '', 1)
87 actual_dataset = data_store.exists(f'.{self.exp_prefix}{actual_name}')
88 if based_dataset and not actual_dataset:
89 name = based_name
90 elif not based_dataset and actual_dataset:
91 name = actual_name
92 elif all((based_dataset, actual_dataset)) and based_dataset != actual_dataset:
93 msg = (
94 f"experiment {name} may previously have logged to {data_store.prefix}{based_name}, "
95 f"now using {data_store.prefix}{actual_name}"
96 )
97 logger.warning(msg)
98 name = actual_name
99 else:
100 # neither data exists, use the actual name
101 name = actual_name
102 # --end fix for #452
103 return tracker.experiment(name) if not raw else tracker
104
105 def drop(self, name, force=False, version=-1, data_store=None, **kwargs):
106 data_store = data_store or self.data_store
107 meta = self.model_store.metadata(name)
108 dataset = meta.attributes.get('tracking', {}).get('dataset')
109 data_store.drop(dataset, force=True) if dataset else None
110 return self.model_store._drop(name, force=force, version=version, **kwargs)