1from os.path import basename
2
3import os
4import re
5
6from omegaml.backends.basedata import BaseDataBackend
7from omegaml.backends.package.packager import install_and_import, RunnablePackageMixin
8
9
[docs]
10class PythonPipSourcedPackageData(RunnablePackageMixin, BaseDataBackend):
11 """
12 Backend to support locally sourced custom scripts deployment to runtimes cluster
13
14 This supports any pip-installable remote source like git, hg, svn, bzr
15
16 Usage::
17
18 # pypi package
19 om.scripts.put('pypi://<pkgname>', '<pkgname>')
20
21 # git hosted
22 om.scripts.put('git+https://github.com/account/repo/<pkgname>', '<pkgname>')
23
24 # retrieve the package and call pip install
25 om.scripts.get('name')
26
27 Notes:
28 * `pypi://<pkgname>` can be any specification supported by pip, e.g.::
29
30 om.scripts.put('pypi://<pkgname==version>', ...)
31 om.scripts.put('git+https://....@tag#egg=<pkgname>', ...)
32
33 * the package specification is stored as ``metadata.kind_meta['pip_source']``
34
35 See Also:
36 * https://packaging.python.org/tutorials/packaging-projects/
37 """
38 KIND = 'pipsrc.package'
39
[docs]
40 @classmethod
41 def supports(self, obj, name, **kwargs):
42 # see https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support
43 # pypi is our own protocol to designate pypi hosted packages
44 # all others are pip supported formats
45 # the pattern supports <source>:// and <source>+<protocol>:// formats
46 # for pypi, .get() will remove the pypi:// part
47 pip_protocols = '|'.join(('git', 'hg', 'svn', 'bzr', 'pypi'))
48 pattern = r'^{}(\+.+)?://.*'
49 is_pip_protocol = lambda v: re.match(pattern.format(pip_protocols), v)
50 return isinstance(obj, str) and is_pip_protocol(obj)
51
[docs]
52 def put(self, obj, name, attributes=None, **kwargs):
53 """
54 save a python package
55
56 This stores a PIP-sourceable package specification in the objects
57 ``Metadata.kind_meta``::
58
59 { 'pip_source': '<obj:package spec>' }
60
61 :param obj: full path to package file or directory, syntax as
62 pkg://path/to/dist.tar.gz or pkg://path/to/setup.py
63 :param name: name of package. must be the actual name of the package
64 as specified in setup.py
65 :return: the Metadata object
66 """
67 kind_meta = {
68 'pip_source': obj,
69 }
70 return self.data_store._make_metadata(
71 name=name,
72 prefix=self.data_store.prefix,
73 bucket=self.data_store.bucket,
74 kind=PythonPipSourcedPackageData.KIND,
75 kind_meta=kind_meta,
76 attributes=attributes).save()
77
[docs]
78 def get(self, name, keep=False, **kwargs):
79 """
80 Load package from store, install it locally and load.
81
82 This runs `pip install <package spec>`, taking the package specification
83 from Metadata.kind_meta['pip-source']. See the put() method for details.
84
85 :param name: the name of the package
86 :param keep: keep the packages load path in sys.path, defaults to False
87 :param kwargs:
88 :return: the loaded module
89 """
90 pkgname = basename(name)
91 meta = self.data_store.metadata(name)
92 dstdir = self.packages_path
93 pip_source = meta.kind_meta['pip_source']
94 # pip does not know about pypi
95 pip_name = pip_source.replace('pypi://', '')
96 mod = install_and_import(pip_name, pkgname, dstdir, keep=keep)
97 return mod
98
99
100 @property
101 def packages_path(self):
102 return os.path.join(self.data_store.tmppath, 'packages')