1import logging
2import os
3
4from omegaml.backends.basemodel import BaseModelBackend
5from omegaml.backends.tracking.base import TrackingProvider
6
7logger = logging.getLogger(__name__)
8
9
[docs]
10class ExperimentBackend(BaseModelBackend):
11 """ ExperimentBackend provides storage of tracker configurations
12
13 Usage:
14
15 To log metrics and other data::
16
17 with om.runtime.experiment('myexp') as exp:
18 om.runtime.model('mymodel').fit(X, Y)
19 om.runtime.model('mymodel').score(X, Y) # automatically log score result
20 exp.log_metric('mymetric', value)
21 exp.log_param('myparam', value)
22 exp.log_artifact(X, 'X')
23 exp.log_artifact(Y, 'Y')
24 exp.log_artifact(om.models.metadata('mymodel'), 'mymodel')
25
26 To log data and automatically profile system data::
27
28 with om.runtime.experiment('myexp', provider='profiling') as exp:
29 om.runtime.model('mymodel').fit(X, Y)
30 om.runtime.model('mymodel').score(X, Y) # automatically log score result
31 exp.log_metric('mymetric', value)
32 exp.log_param('myparam', value)
33 exp.log_artifact(X, 'X')
34 exp.log_artifact(Y, 'Y')
35 exp.log_artifact(om.models.metadata('mymodel'), 'mymodel')
36
37 # profiling data contains metrics for cpu, memory and disk use
38 data = exp.data(event='profile')
39
40 To get back experiment data without running an experiment::
41
42 # recommended way
43 exp = om.runtime.experiment('myexp').use()
44 exp_df = exp.data()
45
46 # experiments exist in the models store
47 exp = om.models.get('experiments/myexp')
48 exp_df = exp.data()
49
50 See Also:
51
52 * :class:`omegaml.backends.tracking.OmegaSimpleTracker`
53 * :class:`omegaml.backends.tracking.OmegaProfilingTracker`
54 """
55 KIND = 'experiment.tracker'
56 exp_prefix = 'experiments/'
57
[docs]
58 @classmethod
59 def supports(self, obj, name, **kwargs):
60 return isinstance(obj, TrackingProvider)
61
[docs]
62 def put(self, obj, name, **kwargs):
63 name = f'{self.exp_prefix}{name}' if not name.startswith(self.exp_prefix) else name
64 # FIXME use proper pickle magic to avoid storing the store
65 store = obj._store
66 obj._store = None
67 obj._model_store = None
68 meta = super().put(obj, name, **kwargs)
69 meta.attributes.setdefault('tracking', {})
70 meta.attributes['tracking']['dataset'] = obj._data_name
71 meta.save()
72 obj._store = store
73 obj._model_store = self.model_store
74 return meta
75
[docs]
76 def get(self, name, raw=False, data_store=None, **kwargs):
77 data_store = data_store or self.data_store
78 assert data_store is not None, "experiments require a datastore, specify data_store=om.datasets"
79 tracker = super().get(name, **kwargs)
80 tracker._store = data_store
81 tracker._model_store = self.model_store
82 # fix for #452, maintain backwards compatibility
83 based_name = os.path.basename(name)
84 based_dataset = data_store.exists(f'.{self.exp_prefix}{based_name}')
85 actual_name = name.replace(self.exp_prefix, '', 1)
86 actual_dataset = data_store.exists(f'.{self.exp_prefix}{actual_name}')
87 if based_dataset and not actual_dataset:
88 name = based_name
89 elif not based_dataset and actual_dataset:
90 name = actual_name
91 elif all((based_dataset, actual_dataset)) and based_dataset != actual_dataset:
92 msg = (f"experiment {name} may previously have logged to {data_store.prefix}{based_name}, "
93 f"now using {data_store.prefix}{actual_name}")
94 logger.warning(msg)
95 name = actual_name
96 else:
97 # neither data exists, use the actual name
98 name = actual_name
99 # --end fix for #452
100 return tracker.experiment(name) if not raw else tracker
101
102 def drop(self, name, force=False, version=-1, data_store=None, **kwargs):
103 data_store = data_store or self.data_store
104 meta = self.model_store.metadata(name)
105 dataset = meta.attributes.get('tracking', {}).get('dataset')
106 data_store.drop(dataset, force=True) if dataset else None
107 return self.model_store._drop(name, force=force, version=version, **kwargs)