1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """objects and functions used in dealing with packages
23 """
24
25 import ihooks
26 import glob
27 import os
28 import sys
29
30 from twisted.python import rebuild, reflect
31
32 from flumotion.common import log, common
33 from flumotion.configure import configure
34
35 __version__ = "$Rev: 7927 $"
36
37
39 """
40 I am overriding ihook's ModuleImporter's import_module() method to
41 accept (and ignore) the 'level' keyword argument that appeared in
42 the built-in __import__() function in python2.5.
43
44 While no built-in modules in python2.5 seem to use that keyword
45 argument, 'encodings' module in python2.6 does and so it breaks if
46 used together with ihooks.
47
48 I make no attempt to properly support the 'level' argument -
49 ihooks didn't make it into py3k, and the only use in python2.6
50 we've seen so far, in 'encodings', serves as a performance hint
51 and it seems that can be ignored with no difference in behaviour.
52 """
53
54 - def import_module(self, name, globals=None, locals=None, fromlist=None,
55 level=-1):
56
57 return ihooks.ModuleImporter.import_module(self, name, globals,
58 locals, fromlist)
59
60
62 """
63 I am an import Hooks object that makes sure that every package that gets
64 loaded has every necessary path in the module's __path__ list.
65
66 @type packager: L{Packager}
67 """
68 packager = None
69
71
72 log.log('packager', 'load_package %s' % name)
73 ret = ihooks.Hooks.load_package(self, name, filename, file)
74
75 m = sys.modules[name]
76
77 packagePaths = self.packager.getPathsForPackage(name)
78 if not packagePaths:
79 return ret
80
81
82 paths = [os.path.join(path, name.replace('.', os.sep))
83 for path in packagePaths]
84 for path in paths:
85 if not path in m.__path__:
86 log.log('packager', 'adding path %s for package %s' % (
87 path, name))
88 m.__path__.append(path)
89
90 return ret
91
92
94 """
95 I am an object through which package paths can be registered, to support
96 the partitioning of the module import namespace across bundles.
97 """
98
99 logCategory = 'packager'
100
102 self._paths = {}
103 self._packages = {}
104 self.install()
105
107 """
108 Install our custom importer that uses bundled packages.
109 """
110 self.debug('installing custom importer')
111 self._hooks = PackageHooks()
112 self._hooks.packager = self
113 if sys.version_info < (2, 6):
114 self._importer = ihooks.ModuleImporter()
115 else:
116 self.debug('python2.6 or later detected - using patched'
117 ' ModuleImporter')
118 self._importer = _PatchedModuleImporter()
119 self._importer.set_hooks(self._hooks)
120 self._importer.install()
121
123 """
124 Return all absolute paths to the top level of a tree from which
125 (part of) the given package name can be imported.
126 """
127 if packageName not in self._packages:
128 return None
129
130 return [self._paths[key] for key in self._packages[packageName]]
131
133 """
134 Register a given path as a path that can be imported from.
135 Used to support partition of bundled code or import code from various
136 uninstalled location.
137
138 sys.path will also be changed to include this, and remove references
139 to older packagePath's for the same bundle.
140
141 @param packagePath: path to add under which the module namespaces live,
142 (ending in an md5sum, for flumotion purposes)
143 @type packagePath: string
144 @param key a unique id for the package being registered
145 @type key: string
146 @param prefix: prefix of the packages to be considered
147 @type prefix: string
148 """
149
150 new = True
151 packagePath = os.path.abspath(packagePath)
152 if not os.path.exists(packagePath):
153 log.warning('bundle',
154 'registering a non-existing package path %s' % packagePath)
155
156 self.log('registering packagePath %s' % packagePath)
157
158
159 if key in self._paths:
160 oldPath = self._paths[key]
161 if packagePath == oldPath:
162 self.log('already registered %s for key %s' % (
163 packagePath, key))
164 return
165 new = False
166
167
168
169
170
171 if not os.path.isdir(packagePath):
172 log.warning('bundle', 'package path not a dir: %s',
173 packagePath)
174 packageNames = []
175 else:
176 packageNames = _findPackageCandidates(packagePath, prefix)
177
178 if not packageNames:
179 log.log('bundle',
180 'packagePath %s does not have candidates starting with %s' %
181 (packagePath, prefix))
182 return
183 packageNames.sort()
184
185 self.log('package candidates %r' % packageNames)
186
187 if not new:
188
189 log.log('bundle',
190 'replacing old path %s with new path %s for key %s' % (
191 oldPath, packagePath, key))
192
193 if oldPath in sys.path:
194 log.log('bundle',
195 'removing old packagePath %s from sys.path' % oldPath)
196 sys.path.remove(oldPath)
197
198
199 for keys in self._packages.values():
200 if key in keys:
201 keys.remove(key)
202
203 self._paths[key] = packagePath
204
205
206 if not packagePath in sys.path:
207 self.log('adding packagePath %s to sys.path' % packagePath)
208 sys.path.insert(0, packagePath)
209
210
211 for name in packageNames:
212 if name not in self._packages:
213 self._packages[name] = [key]
214 else:
215 self._packages[name].insert(0, key)
216
217 self.log('packagePath %s has packageNames %r' % (
218 packagePath, packageNames))
219
220
221 packageNames.reverse()
222
223 for packageName in packageNames:
224 if packageName not in sys.modules:
225 continue
226 self.log('fixing up %s ...' % packageName)
227
228
229 package = sys.modules.get(packageName)
230 for path in package.__path__:
231 if not new and path.startswith(oldPath):
232 self.log('%s.__path__ before remove %r' % (
233 packageName, package.__path__))
234 self.log('removing old %s from %s.__path__' % (
235 path, name))
236 package.__path__.remove(path)
237 self.log('%s.__path__ after remove %r' % (
238 packageName, package.__path__))
239
240
241
242
243
244 newPath = os.path.join(packagePath,
245 packageName.replace('.', os.sep))
246
247
248
249
250 if len(package.__path__) == 0:
251
252
253
254
255 self.debug('WARN: package %s does not have __path__ values' % (
256 packageName))
257 elif package.__path__[0] == newPath:
258 self.log('path %s already at start of %s.__path__' % (
259 newPath, packageName))
260 continue
261
262 if newPath in package.__path__:
263 package.__path__.remove(newPath)
264 self.log('moving %s to front of %s.__path__' % (
265 newPath, packageName))
266 else:
267 self.log('inserting new %s into %s.__path__' % (
268 newPath, packageName))
269 package.__path__.insert(0, newPath)
270
271
272
273
274
275
276
277
278 self.log('fixed up %s, __path__ %s ...' % (
279 packageName, package.__path__))
280
281
282
283 if not new:
284 self.log('finding end module candidates')
285 if not os.path.isdir(packagePath):
286 log.warning('bundle', 'package path not a dir: %s',
287 path)
288 moduleNames = []
289 else:
290 moduleNames = findEndModuleCandidates(packagePath, prefix)
291 self.log('end module candidates to rebuild: %r' % moduleNames)
292 for name in moduleNames:
293 if name in sys.modules:
294
295 self.log("rebuilding non-package module %s" % name)
296 try:
297 module = reflect.namedAny(name)
298 except AttributeError:
299 log.warning('bundle',
300 "could not reflect non-package module %s" % name)
301 continue
302
303 if hasattr(module, '__path__'):
304 self.log('rebuilding module %s with paths %r' % (name,
305 module.__path__))
306 rebuild.rebuild(module)
307
308
309
310 self.log('registered packagePath %s for key %s' % (packagePath, key))
311
313 """
314 Unregister all previously registered package paths, and uninstall
315 the custom importer.
316 """
317 for path in self._paths.values():
318 if path in sys.path:
319 self.log('removing packagePath %s from sys.path' % path)
320 sys.path.remove(path)
321 self._paths = {}
322 self._packages = {}
323 self.debug('uninstalling custom importer')
324 self._importer.uninstall()
325
326
328 """
329 I'm similar to os.listdir, but I work recursively and only return
330 directories containing python code.
331
332 @param path: the path
333 @type path: string
334 """
335 retval = []
336 try:
337 files = os.listdir(path)
338 except OSError:
339 pass
340 else:
341 for f in files:
342
343 p = os.path.join(path, f)
344 if os.path.isdir(p) and f != '.svn':
345 retval += _listDirRecursively(p)
346
347 if glob.glob(os.path.join(path, '*.py*')):
348 retval.append(path)
349
350 return retval
351
352
354 """
355 I'm similar to os.listdir, but I work recursively and only return
356 files representing python non-package modules.
357
358 @param path: the path
359 @type path: string
360
361 @rtype: list
362 @returns: list of files underneath the given path containing python code
363 """
364 retval = []
365
366
367 dirs = _listDirRecursively(path)
368
369 for directory in dirs:
370 pyfiles = glob.glob(os.path.join(directory, '*.py*'))
371 dontkeep = glob.glob(os.path.join(directory, '*__init__.py*'))
372 for f in dontkeep:
373 if f in pyfiles:
374 pyfiles.remove(f)
375
376 retval.extend(pyfiles)
377
378 return retval
379
380
382 """
383 I take a directory and return a list of candidate python packages
384 under that directory that start with the given prefix.
385 A package is a module containing modules; typically the directory
386 with the same name as the package contains __init__.py
387
388 @param path: the path
389 @type path: string
390 """
391
392
393 dirs = _listDirRecursively(os.path.join(path, prefix))
394
395
396 bundlePaths = [x[len(path) + 1:] for x in dirs]
397
398
399 bundlePaths = [path for path in bundlePaths if path.find('.svn') == -1]
400 bundlePaths = [path for path in bundlePaths if path.find('-') == -1]
401
402
403 bundlePackages = [".".join(x.split(os.path.sep)) for x in bundlePaths]
404
405
406
407 packages = {}
408 for name in bundlePackages:
409 packages[name] = 1
410 parts = name.split(".")
411 build = None
412 for p in parts:
413 if not build:
414 build = p
415 else:
416 build = build + "." + p
417 packages[build] = 1
418
419 bundlePackages = packages.keys()
420
421
422 bundlePackages.sort()
423
424 return bundlePackages
425
426
428 """
429 I take a directory and return a list of candidate python end modules
430 (i.e., non-package modules) for the given module prefix.
431
432 @param path: the path under which to search for end modules
433 @type path: string
434 @param prefix: module prefix to check candidates under
435 @type prefix: string
436 """
437 pathPrefix = "/".join(prefix.split("."))
438 files = _listPyFileRecursively(os.path.join(path, pathPrefix))
439
440
441 importPaths = [x[len(path) + 1:] for x in files]
442
443
444 importPaths = [path for path in importPaths if path.find('.svn') == -1]
445 importPaths = [path for path in importPaths if path.find('-') == -1]
446
447
448 endModules = [common.pathToModuleName(x) for x in importPaths]
449
450
451 endModules = [module for module in endModules
452 if module and module.startswith(prefix)]
453
454
455 endModules.sort()
456
457
458 res = {}
459 for b in endModules:
460 res[b] = 1
461
462 return res.keys()
463
464
465 __packager = None
466
467
479