1import os
2import re
3from os.path import basename
4
5from omegaml.backends.basedata import BaseDataBackend
6from omegaml.backends.package.packager import RunnablePackageMixin, install_and_import
7
8
[docs]
9class PythonPipSourcedPackageData(RunnablePackageMixin, BaseDataBackend):
10 """
11 Backend to support locally sourced custom scripts deployment to runtimes cluster
12
13 This supports any pip-installable remote source like git, hg, svn, bzr
14
15 Usage::
16
17 # pypi package
18 om.scripts.put('pypi://<pkgname>', '<pkgname>')
19
20 # git hosted
21 om.scripts.put('git+https://github.com/account/repo/<pkgname>', '<pkgname>')
22
23 # retrieve the package and call pip install
24 om.scripts.get('name')
25
26 Notes:
27 * `pypi://<pkgname>` can be any specification supported by pip, e.g.::
28
29 om.scripts.put('pypi://<pkgname==version>', ...)
30 om.scripts.put('git+https://....@tag#egg=<pkgname>', ...)
31
32 * the package specification is stored as ``metadata.kind_meta['pip_source']``
33
34 See Also:
35 * https://packaging.python.org/tutorials/packaging-projects/
36 """
37
38 KIND = 'pipsrc.package'
39
[docs]
40 @classmethod
41 def supports(self, obj, name, **kwargs):
42 # see https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support
43 # pypi is our own protocol to designate pypi hosted packages
44 # all others are pip supported formats
45 # the pattern supports <source>:// and <source>+<protocol>:// formats
46 # for pypi, .get() will remove the pypi:// part
47 pip_protocols = '|'.join(('git', 'hg', 'svn', 'bzr', 'pypi'))
48 pattern = r'^{}(\+.+)?://.*'
49 is_pip_protocol = lambda v: re.match(pattern.format(pip_protocols), v)
50 return isinstance(obj, str) and is_pip_protocol(obj)
51
[docs]
52 def put(self, obj, name, attributes=None, **kwargs):
53 """
54 save a python package
55
56 This stores a PIP-sourceable package specification in the objects
57 ``Metadata.kind_meta``::
58
59 { 'pip_source': '<obj:package spec>' }
60
61 :param obj: full path to package file or directory, syntax as
62 pkg://path/to/dist.tar.gz or pkg://path/to/setup.py
63 :param name: name of package. must be the actual name of the package
64 as specified in setup.py
65 :return: the Metadata object
66 """
67 kind_meta = {
68 'pip_source': obj,
69 }
70 return self.data_store._make_metadata(
71 name=name,
72 prefix=self.data_store.prefix,
73 bucket=self.data_store.bucket,
74 kind=PythonPipSourcedPackageData.KIND,
75 kind_meta=kind_meta,
76 attributes=attributes,
77 ).save()
78
[docs]
79 def get(self, name, keep=False, **kwargs):
80 """
81 Load package from store, install it locally and load.
82
83 This runs `pip install <package spec>`, taking the package specification
84 from Metadata.kind_meta['pip-source']. See the put() method for details.
85
86 :param name: the name of the package
87 :param keep: keep the packages load path in sys.path, defaults to False
88 :param kwargs:
89 :return: the loaded module
90 """
91 pkgname = basename(name)
92 meta = self.data_store.metadata(name)
93 dstdir = self.packages_path
94 pip_source = meta.kind_meta['pip_source']
95 # pip does not know about pypi
96 pip_name = pip_source.replace('pypi://', '')
97 mod = install_and_import(pip_name, pkgname, dstdir, keep=keep)
98 return mod
99
100 @property
101 def packages_path(self):
102 return os.path.join(self.data_store.tmppath, 'packages')