1from os.path import basename, dirname
2
3import os
4
5from omegaml.backends.basedata import BaseDataBackend
6from omegaml.backends.package.packager import build_sdist, install_and_import, load_from_path, RunnablePackageMixin
7
8
[docs]
9class PythonPackageData(RunnablePackageMixin, BaseDataBackend):
10 """
11 Backend to support locally sourced custom scripts deployment to runtimes cluster
12
13 This supports any local setup.py
14
15 Usage::
16
17 om.scripts.put('pkg://path/to/setup.py', 'myname')
18 om.scripts.get('myname')
19
20 Note that setup.py must minimally specify the following. The
21 name and packages kwargs must specify the same name as given
22 in put(..., name)
23
24 # setup.py
25 from setuptools import setup
26 setup(name='myname', packages=['myname'])
27
28 See Also:
29 https://packaging.python.org/tutorials/packaging-projects/
30 """
31 KIND = 'python.package'
32
[docs]
33 @classmethod
34 def supports(self, obj, name, **kwargs):
35 return isinstance(obj, str) and obj.startswith('pkg://')
36
[docs]
37 def put(self, obj, name, attributes=None, **kwargs):
38 """
39 save a python package
40
41 This takes the full path to a setuptools setup.py, or a directory
42 containing a setup.py file. It then executes `python setup.py sdist`
43 and stores the resulting .tar.gz file in om.scripts
44
45 :param obj: full path to package file or directory, syntax as
46 pkg://path/to/dist.tar.gz or pkg://path/to/setup.py
47 :param name: name of package. must be the actual name of the package
48 as specified in setup.py
49 :return: the Metadata object
50 """
51 pkgsrc = pkgdist = obj.split('//')[1]
52 if not 'tar.gz' in os.path.basename(pkgdist):
53 pkgsrc = pkgsrc.replace('setup.py', '')
54 distdir = os.path.join(pkgsrc, 'dist')
55 pkgdist = build_sdist(pkgsrc, distdir)
56 filename = self.data_store.object_store_key(name, 'pkg', hashed=True)
57 gridfile = self._store_to_file(self.data_store, pkgdist, filename)
58 return self.data_store._make_metadata(
59 name=name,
60 prefix=self.data_store.prefix,
61 bucket=self.data_store.bucket,
62 kind=PythonPackageData.KIND,
63 attributes=attributes,
64 gridfile=gridfile).save()
65
[docs]
66 def get(self, name, localpath=None, keep=False, install=True, **kwargs):
67 """
68 Load package from store, install it locally and load.
69
70 This copies the package's .tar.gz file from om.scripts to a local temp
71 path and runs `pip install` on it.
72
73 :param name: the name of the package
74 :param keep: keep the packages load path in sys.path, defaults to False
75 :param localpath: the local path to store the package
76 :param install: if True call pip install on the retrieved package
77 :param kwargs:
78 :return: the loaded module
79 """
80 pkgname = basename(name)
81 packagefname = '{}.tar.gz'.format(os.path.join(localpath or self.data_store.tmppath, pkgname))
82 os.makedirs(dirname(packagefname), exist_ok=True)
83 self.path = self.packages_path
84 dstdir = localpath or self.path
85 if not os.path.exists(os.path.join(dstdir, pkgname)):
86 meta = self.data_store.metadata(name)
87 outf = meta.gridfile
88 with open(packagefname, 'wb') as pkgf:
89 pkgf.write(outf.read())
90 if install:
91 mod = install_and_import(packagefname, pkgname, dstdir, keep=keep)
92 else:
93 mod = packagefname
94 elif install:
95 mod = load_from_path(pkgname, dstdir, keep=keep)
96 else:
97 mod = os.path.join(dstdir, pkgname)
98 return mod
99
100 @property
101 def packages_path(self):
102 return os.path.join(self.data_store.tmppath, 'packages')