util.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290
  1. """Utility code for constructing importers, etc."""
  2. from . import abc
  3. from ._bootstrap import module_from_spec
  4. from ._bootstrap import _resolve_name
  5. from ._bootstrap import spec_from_loader
  6. from ._bootstrap import _find_spec
  7. from ._bootstrap_external import MAGIC_NUMBER
  8. from ._bootstrap_external import cache_from_source
  9. from ._bootstrap_external import decode_source
  10. from ._bootstrap_external import source_from_cache
  11. from ._bootstrap_external import spec_from_file_location
  12. from contextlib import contextmanager
  13. import functools
  14. import sys
  15. import types
  16. import warnings
  17. def resolve_name(name, package):
  18. """Resolve a relative module name to an absolute one."""
  19. if not name.startswith('.'):
  20. return name
  21. elif not package:
  22. raise ValueError('{!r} is not a relative name '
  23. '(no leading dot)'.format(name))
  24. level = 0
  25. for character in name:
  26. if character != '.':
  27. break
  28. level += 1
  29. return _resolve_name(name[level:], package, level)
  30. def _find_spec_from_path(name, path=None):
  31. """Return the spec for the specified module.
  32. First, sys.modules is checked to see if the module was already imported. If
  33. so, then sys.modules[name].__spec__ is returned. If that happens to be
  34. set to None, then ValueError is raised. If the module is not in
  35. sys.modules, then sys.meta_path is searched for a suitable spec with the
  36. value of 'path' given to the finders. None is returned if no spec could
  37. be found.
  38. Dotted names do not have their parent packages implicitly imported. You will
  39. most likely need to explicitly import all parent packages in the proper
  40. order for a submodule to get the correct spec.
  41. """
  42. if name not in sys.modules:
  43. return _find_spec(name, path)
  44. else:
  45. module = sys.modules[name]
  46. if module is None:
  47. return None
  48. try:
  49. spec = module.__spec__
  50. except AttributeError:
  51. raise ValueError('{}.__spec__ is not set'.format(name)) from None
  52. else:
  53. if spec is None:
  54. raise ValueError('{}.__spec__ is None'.format(name))
  55. return spec
  56. def find_spec(name, package=None):
  57. """Return the spec for the specified module.
  58. First, sys.modules is checked to see if the module was already imported. If
  59. so, then sys.modules[name].__spec__ is returned. If that happens to be
  60. set to None, then ValueError is raised. If the module is not in
  61. sys.modules, then sys.meta_path is searched for a suitable spec with the
  62. value of 'path' given to the finders. None is returned if no spec could
  63. be found.
  64. If the name is for submodule (contains a dot), the parent module is
  65. automatically imported.
  66. The name and package arguments work the same as importlib.import_module().
  67. In other words, relative module names (with leading dots) work.
  68. """
  69. fullname = resolve_name(name, package) if name.startswith('.') else name
  70. if fullname not in sys.modules:
  71. parent_name = fullname.rpartition('.')[0]
  72. if parent_name:
  73. # Use builtins.__import__() in case someone replaced it.
  74. parent = __import__(parent_name, fromlist=['__path__'])
  75. return _find_spec(fullname, parent.__path__)
  76. else:
  77. return _find_spec(fullname, None)
  78. else:
  79. module = sys.modules[fullname]
  80. if module is None:
  81. return None
  82. try:
  83. spec = module.__spec__
  84. except AttributeError:
  85. raise ValueError('{}.__spec__ is not set'.format(name)) from None
  86. else:
  87. if spec is None:
  88. raise ValueError('{}.__spec__ is None'.format(name))
  89. return spec
  90. @contextmanager
  91. def _module_to_load(name):
  92. is_reload = name in sys.modules
  93. module = sys.modules.get(name)
  94. if not is_reload:
  95. # This must be done before open() is called as the 'io' module
  96. # implicitly imports 'locale' and would otherwise trigger an
  97. # infinite loop.
  98. module = type(sys)(name)
  99. # This must be done before putting the module in sys.modules
  100. # (otherwise an optimization shortcut in import.c becomes wrong)
  101. module.__initializing__ = True
  102. sys.modules[name] = module
  103. try:
  104. yield module
  105. except Exception:
  106. if not is_reload:
  107. try:
  108. del sys.modules[name]
  109. except KeyError:
  110. pass
  111. finally:
  112. module.__initializing__ = False
  113. def set_package(fxn):
  114. """Set __package__ on the returned module.
  115. This function is deprecated.
  116. """
  117. @functools.wraps(fxn)
  118. def set_package_wrapper(*args, **kwargs):
  119. warnings.warn('The import system now takes care of this automatically.',
  120. DeprecationWarning, stacklevel=2)
  121. module = fxn(*args, **kwargs)
  122. if getattr(module, '__package__', None) is None:
  123. module.__package__ = module.__name__
  124. if not hasattr(module, '__path__'):
  125. module.__package__ = module.__package__.rpartition('.')[0]
  126. return module
  127. return set_package_wrapper
  128. def set_loader(fxn):
  129. """Set __loader__ on the returned module.
  130. This function is deprecated.
  131. """
  132. @functools.wraps(fxn)
  133. def set_loader_wrapper(self, *args, **kwargs):
  134. warnings.warn('The import system now takes care of this automatically.',
  135. DeprecationWarning, stacklevel=2)
  136. module = fxn(self, *args, **kwargs)
  137. if getattr(module, '__loader__', None) is None:
  138. module.__loader__ = self
  139. return module
  140. return set_loader_wrapper
  141. def module_for_loader(fxn):
  142. """Decorator to handle selecting the proper module for loaders.
  143. The decorated function is passed the module to use instead of the module
  144. name. The module passed in to the function is either from sys.modules if
  145. it already exists or is a new module. If the module is new, then __name__
  146. is set the first argument to the method, __loader__ is set to self, and
  147. __package__ is set accordingly (if self.is_package() is defined) will be set
  148. before it is passed to the decorated function (if self.is_package() does
  149. not work for the module it will be set post-load).
  150. If an exception is raised and the decorator created the module it is
  151. subsequently removed from sys.modules.
  152. The decorator assumes that the decorated function takes the module name as
  153. the second argument.
  154. """
  155. warnings.warn('The import system now takes care of this automatically.',
  156. DeprecationWarning, stacklevel=2)
  157. @functools.wraps(fxn)
  158. def module_for_loader_wrapper(self, fullname, *args, **kwargs):
  159. with _module_to_load(fullname) as module:
  160. module.__loader__ = self
  161. try:
  162. is_package = self.is_package(fullname)
  163. except (ImportError, AttributeError):
  164. pass
  165. else:
  166. if is_package:
  167. module.__package__ = fullname
  168. else:
  169. module.__package__ = fullname.rpartition('.')[0]
  170. # If __package__ was not set above, __import__() will do it later.
  171. return fxn(self, module, *args, **kwargs)
  172. return module_for_loader_wrapper
  173. class _Module(types.ModuleType):
  174. """A subclass of the module type to allow __class__ manipulation."""
  175. class _LazyModule(types.ModuleType):
  176. """A subclass of the module type which triggers loading upon attribute access."""
  177. def __getattribute__(self, attr):
  178. """Trigger the load of the module and return the attribute."""
  179. # All module metadata must be garnered from __spec__ in order to avoid
  180. # using mutated values.
  181. # Stop triggering this method.
  182. self.__class__ = _Module
  183. # Get the original name to make sure no object substitution occurred
  184. # in sys.modules.
  185. original_name = self.__spec__.name
  186. # Figure out exactly what attributes were mutated between the creation
  187. # of the module and now.
  188. attrs_then = self.__spec__.loader_state
  189. attrs_now = self.__dict__
  190. attrs_updated = {}
  191. for key, value in attrs_now.items():
  192. # Code that set the attribute may have kept a reference to the
  193. # assigned object, making identity more important than equality.
  194. if key not in attrs_then:
  195. attrs_updated[key] = value
  196. elif id(attrs_now[key]) != id(attrs_then[key]):
  197. attrs_updated[key] = value
  198. self.__spec__.loader.exec_module(self)
  199. # If exec_module() was used directly there is no guarantee the module
  200. # object was put into sys.modules.
  201. if original_name in sys.modules:
  202. if id(self) != id(sys.modules[original_name]):
  203. msg = ('module object for {!r} substituted in sys.modules '
  204. 'during a lazy load')
  205. raise ValueError(msg.format(original_name))
  206. # Update after loading since that's what would happen in an eager
  207. # loading situation.
  208. self.__dict__.update(attrs_updated)
  209. return getattr(self, attr)
  210. def __delattr__(self, attr):
  211. """Trigger the load and then perform the deletion."""
  212. # To trigger the load and raise an exception if the attribute
  213. # doesn't exist.
  214. self.__getattribute__(attr)
  215. delattr(self, attr)
  216. class LazyLoader(abc.Loader):
  217. """A loader that creates a module which defers loading until attribute access."""
  218. @staticmethod
  219. def __check_eager_loader(loader):
  220. if not hasattr(loader, 'exec_module'):
  221. raise TypeError('loader must define exec_module()')
  222. @classmethod
  223. def factory(cls, loader):
  224. """Construct a callable which returns the eager loader made lazy."""
  225. cls.__check_eager_loader(loader)
  226. return lambda *args, **kwargs: cls(loader(*args, **kwargs))
  227. def __init__(self, loader):
  228. self.__check_eager_loader(loader)
  229. self.loader = loader
  230. def create_module(self, spec):
  231. """Create a module which can have its __class__ manipulated."""
  232. return _Module(spec.name)
  233. def exec_module(self, module):
  234. """Make the module load lazily."""
  235. module.__spec__.loader = self.loader
  236. module.__loader__ = self.loader
  237. # Don't need to worry about deep-copying as trying to set an attribute
  238. # on an object would have triggered the load,
  239. # e.g. ``module.__spec__.loader = None`` would trigger a load from
  240. # trying to access module.__spec__.
  241. module.__spec__.loader_state = module.__dict__.copy()
  242. module.__class__ = _LazyModule