1import os
2from os.path import basename, dirname
3
4from omegaml.backends.basedata import BaseDataBackend
5from omegaml.backends.package.packager import RunnablePackageMixin, build_sdist, install_and_import, load_from_path
6
7
[docs]
8class PythonPackageData(RunnablePackageMixin, BaseDataBackend):
9 """
10 Backend to support locally sourced custom scripts deployment to runtimes cluster
11
12 This supports any local setup.py
13
14 Usage::
15
16 om.scripts.put('pkg://path/to/setup.py', 'myname')
17 om.scripts.get('myname')
18
19 Note that setup.py must minimally specify the following. The
20 name and packages kwargs must specify the same name as given
21 in put(..., name)
22
23 # setup.py
24 from setuptools import setup
25 setup(name='myname', packages=['myname'])
26
27 See Also:
28 https://packaging.python.org/tutorials/packaging-projects/
29 """
30
31 KIND = 'python.package'
32
[docs]
33 @classmethod
34 def supports(self, obj, name, **kwargs):
35 return isinstance(obj, str) and obj.startswith('pkg://')
36
[docs]
37 def put(self, obj, name, attributes=None, **kwargs):
38 """
39 save a python package
40
41 This takes the full path to a setuptools setup.py, or a directory
42 containing a setup.py file. It then executes `python setup.py sdist`
43 and stores the resulting .tar.gz file in om.scripts
44
45 :param obj: full path to package file or directory, syntax as
46 pkg://path/to/dist.tar.gz or pkg://path/to/setup.py
47 :param name: name of package. must be the actual name of the package
48 as specified in setup.py
49 :return: the Metadata object
50 """
51 pkgsrc = pkgdist = obj.split('//')[1]
52 if not 'tar.gz' in os.path.basename(pkgdist):
53 pkgsrc = pkgsrc.replace('setup.py', '')
54 distdir = os.path.join(pkgsrc, 'dist')
55 pkgdist = build_sdist(pkgsrc, distdir)
56 filename = self.data_store.object_store_key(name, 'pkg', hashed=True)
57 gridfile = self._store_to_file(self.data_store, pkgdist, filename)
58 return self.data_store._make_metadata(
59 name=name,
60 prefix=self.data_store.prefix,
61 bucket=self.data_store.bucket,
62 kind=PythonPackageData.KIND,
63 attributes=attributes,
64 gridfile=gridfile,
65 ).save()
66
[docs]
67 def get(self, name, localpath=None, keep=False, install=True, **kwargs):
68 """
69 Load package from store, install it locally and load.
70
71 This copies the package's .tar.gz file from om.scripts to a local temp
72 path and runs `pip install` on it.
73
74 :param name: the name of the package
75 :param keep: keep the packages load path in sys.path, defaults to False
76 :param localpath: the local path to store the package
77 :param install: if True call pip install on the retrieved package
78 :param kwargs:
79 :return: the loaded module
80 """
81 pkgname = basename(name)
82 packagefname = f'{os.path.join(localpath or self.data_store.tmppath, pkgname)}.tar.gz'
83 os.makedirs(dirname(packagefname), exist_ok=True)
84 self.path = self.packages_path
85 dstdir = localpath or self.path
86 if not os.path.exists(os.path.join(dstdir, pkgname)):
87 meta = self.data_store.metadata(name)
88 outf = meta.gridfile
89 with open(packagefname, 'wb') as pkgf:
90 pkgf.write(outf.read())
91 if install:
92 mod = install_and_import(packagefname, pkgname, dstdir, keep=keep)
93 else:
94 mod = packagefname
95 elif install:
96 mod = load_from_path(pkgname, dstdir, keep=keep)
97 else:
98 mod = os.path.join(dstdir, pkgname)
99 return mod
100
101 @property
102 def packages_path(self):
103 return os.path.join(self.data_store.tmppath, 'packages')