__init__.py 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239
  1. '''This module implements specialized container datatypes providing
  2. alternatives to Python's general purpose built-in containers, dict,
  3. list, set, and tuple.
  4. * namedtuple factory function for creating tuple subclasses with named fields
  5. * deque list-like container with fast appends and pops on either end
  6. * ChainMap dict-like class for creating a single view of multiple mappings
  7. * Counter dict subclass for counting hashable objects
  8. * OrderedDict dict subclass that remembers the order entries were added
  9. * defaultdict dict subclass that calls a factory function to supply missing values
  10. * UserDict wrapper around dictionary objects for easier dict subclassing
  11. * UserList wrapper around list objects for easier list subclassing
  12. * UserString wrapper around string objects for easier string subclassing
  13. '''
  14. __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList',
  15. 'UserString', 'Counter', 'OrderedDict', 'ChainMap']
  16. # For backwards compatibility, continue to make the collections ABCs
  17. # available through the collections module.
  18. from _collections_abc import *
  19. import _collections_abc
  20. __all__ += _collections_abc.__all__
  21. from operator import itemgetter as _itemgetter, eq as _eq
  22. from keyword import iskeyword as _iskeyword
  23. import sys as _sys
  24. import heapq as _heapq
  25. from _weakref import proxy as _proxy
  26. from itertools import repeat as _repeat, chain as _chain, starmap as _starmap
  27. from reprlib import recursive_repr as _recursive_repr
  28. try:
  29. from _collections import deque
  30. except ImportError:
  31. pass
  32. else:
  33. MutableSequence.register(deque)
  34. try:
  35. from _collections import defaultdict
  36. except ImportError:
  37. pass
  38. ################################################################################
  39. ### OrderedDict
  40. ################################################################################
  41. class _OrderedDictKeysView(KeysView):
  42. def __reversed__(self):
  43. yield from reversed(self._mapping)
  44. class _OrderedDictItemsView(ItemsView):
  45. def __reversed__(self):
  46. for key in reversed(self._mapping):
  47. yield (key, self._mapping[key])
  48. class _OrderedDictValuesView(ValuesView):
  49. def __reversed__(self):
  50. for key in reversed(self._mapping):
  51. yield self._mapping[key]
  52. class _Link(object):
  53. __slots__ = 'prev', 'next', 'key', '__weakref__'
  54. class OrderedDict(dict):
  55. 'Dictionary that remembers insertion order'
  56. # An inherited dict maps keys to values.
  57. # The inherited dict provides __getitem__, __len__, __contains__, and get.
  58. # The remaining methods are order-aware.
  59. # Big-O running times for all methods are the same as regular dictionaries.
  60. # The internal self.__map dict maps keys to links in a doubly linked list.
  61. # The circular doubly linked list starts and ends with a sentinel element.
  62. # The sentinel element never gets deleted (this simplifies the algorithm).
  63. # The sentinel is in self.__hardroot with a weakref proxy in self.__root.
  64. # The prev links are weakref proxies (to prevent circular references).
  65. # Individual links are kept alive by the hard reference in self.__map.
  66. # Those hard references disappear when a key is deleted from an OrderedDict.
  67. def __init__(*args, **kwds):
  68. '''Initialize an ordered dictionary. The signature is the same as
  69. regular dictionaries, but keyword arguments are not recommended because
  70. their insertion order is arbitrary.
  71. '''
  72. if not args:
  73. raise TypeError("descriptor '__init__' of 'OrderedDict' object "
  74. "needs an argument")
  75. self, *args = args
  76. if len(args) > 1:
  77. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  78. try:
  79. self.__root
  80. except AttributeError:
  81. self.__hardroot = _Link()
  82. self.__root = root = _proxy(self.__hardroot)
  83. root.prev = root.next = root
  84. self.__map = {}
  85. self.__update(*args, **kwds)
  86. def __setitem__(self, key, value,
  87. dict_setitem=dict.__setitem__, proxy=_proxy, Link=_Link):
  88. 'od.__setitem__(i, y) <==> od[i]=y'
  89. # Setting a new item creates a new link at the end of the linked list,
  90. # and the inherited dictionary is updated with the new key/value pair.
  91. if key not in self:
  92. self.__map[key] = link = Link()
  93. root = self.__root
  94. last = root.prev
  95. link.prev, link.next, link.key = last, root, key
  96. last.next = link
  97. root.prev = proxy(link)
  98. dict_setitem(self, key, value)
  99. def __delitem__(self, key, dict_delitem=dict.__delitem__):
  100. 'od.__delitem__(y) <==> del od[y]'
  101. # Deleting an existing item uses self.__map to find the link which gets
  102. # removed by updating the links in the predecessor and successor nodes.
  103. dict_delitem(self, key)
  104. link = self.__map.pop(key)
  105. link_prev = link.prev
  106. link_next = link.next
  107. link_prev.next = link_next
  108. link_next.prev = link_prev
  109. link.prev = None
  110. link.next = None
  111. def __iter__(self):
  112. 'od.__iter__() <==> iter(od)'
  113. # Traverse the linked list in order.
  114. root = self.__root
  115. curr = root.next
  116. while curr is not root:
  117. yield curr.key
  118. curr = curr.next
  119. def __reversed__(self):
  120. 'od.__reversed__() <==> reversed(od)'
  121. # Traverse the linked list in reverse order.
  122. root = self.__root
  123. curr = root.prev
  124. while curr is not root:
  125. yield curr.key
  126. curr = curr.prev
  127. def clear(self):
  128. 'od.clear() -> None. Remove all items from od.'
  129. root = self.__root
  130. root.prev = root.next = root
  131. self.__map.clear()
  132. dict.clear(self)
  133. def popitem(self, last=True):
  134. '''od.popitem() -> (k, v), return and remove a (key, value) pair.
  135. Pairs are returned in LIFO order if last is true or FIFO order if false.
  136. '''
  137. if not self:
  138. raise KeyError('dictionary is empty')
  139. root = self.__root
  140. if last:
  141. link = root.prev
  142. link_prev = link.prev
  143. link_prev.next = root
  144. root.prev = link_prev
  145. else:
  146. link = root.next
  147. link_next = link.next
  148. root.next = link_next
  149. link_next.prev = root
  150. key = link.key
  151. del self.__map[key]
  152. value = dict.pop(self, key)
  153. return key, value
  154. def move_to_end(self, key, last=True):
  155. '''Move an existing element to the end (or beginning if last==False).
  156. Raises KeyError if the element does not exist.
  157. When last=True, acts like a fast version of self[key]=self.pop(key).
  158. '''
  159. link = self.__map[key]
  160. link_prev = link.prev
  161. link_next = link.next
  162. link_prev.next = link_next
  163. link_next.prev = link_prev
  164. root = self.__root
  165. if last:
  166. last = root.prev
  167. link.prev = last
  168. link.next = root
  169. last.next = root.prev = link
  170. else:
  171. first = root.next
  172. link.prev = root
  173. link.next = first
  174. root.next = first.prev = link
  175. def __sizeof__(self):
  176. sizeof = _sys.getsizeof
  177. n = len(self) + 1 # number of links including root
  178. size = sizeof(self.__dict__) # instance dictionary
  179. size += sizeof(self.__map) * 2 # internal dict and inherited dict
  180. size += sizeof(self.__hardroot) * n # link objects
  181. size += sizeof(self.__root) * n # proxy objects
  182. return size
  183. update = __update = MutableMapping.update
  184. def keys(self):
  185. "D.keys() -> a set-like object providing a view on D's keys"
  186. return _OrderedDictKeysView(self)
  187. def items(self):
  188. "D.items() -> a set-like object providing a view on D's items"
  189. return _OrderedDictItemsView(self)
  190. def values(self):
  191. "D.values() -> an object providing a view on D's values"
  192. return _OrderedDictValuesView(self)
  193. __ne__ = MutableMapping.__ne__
  194. __marker = object()
  195. def pop(self, key, default=__marker):
  196. '''od.pop(k[,d]) -> v, remove specified key and return the corresponding
  197. value. If key is not found, d is returned if given, otherwise KeyError
  198. is raised.
  199. '''
  200. if key in self:
  201. result = self[key]
  202. del self[key]
  203. return result
  204. if default is self.__marker:
  205. raise KeyError(key)
  206. return default
  207. def setdefault(self, key, default=None):
  208. 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
  209. if key in self:
  210. return self[key]
  211. self[key] = default
  212. return default
  213. @_recursive_repr()
  214. def __repr__(self):
  215. 'od.__repr__() <==> repr(od)'
  216. if not self:
  217. return '%s()' % (self.__class__.__name__,)
  218. return '%s(%r)' % (self.__class__.__name__, list(self.items()))
  219. def __reduce__(self):
  220. 'Return state information for pickling'
  221. inst_dict = vars(self).copy()
  222. for k in vars(OrderedDict()):
  223. inst_dict.pop(k, None)
  224. return self.__class__, (), inst_dict or None, None, iter(self.items())
  225. def copy(self):
  226. 'od.copy() -> a shallow copy of od'
  227. return self.__class__(self)
  228. @classmethod
  229. def fromkeys(cls, iterable, value=None):
  230. '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S.
  231. If not specified, the value defaults to None.
  232. '''
  233. self = cls()
  234. for key in iterable:
  235. self[key] = value
  236. return self
  237. def __eq__(self, other):
  238. '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
  239. while comparison to a regular mapping is order-insensitive.
  240. '''
  241. if isinstance(other, OrderedDict):
  242. return dict.__eq__(self, other) and all(map(_eq, self, other))
  243. return dict.__eq__(self, other)
  244. try:
  245. from _collections import OrderedDict
  246. except ImportError:
  247. # Leave the pure Python version in place.
  248. pass
  249. ################################################################################
  250. ### namedtuple
  251. ################################################################################
  252. _class_template = """\
  253. from builtins import property as _property, tuple as _tuple
  254. from operator import itemgetter as _itemgetter
  255. from collections import OrderedDict
  256. class {typename}(tuple):
  257. '{typename}({arg_list})'
  258. __slots__ = ()
  259. _fields = {field_names!r}
  260. def __new__(_cls, {arg_list}):
  261. 'Create new instance of {typename}({arg_list})'
  262. return _tuple.__new__(_cls, ({arg_list}))
  263. @classmethod
  264. def _make(cls, iterable, new=tuple.__new__, len=len):
  265. 'Make a new {typename} object from a sequence or iterable'
  266. result = new(cls, iterable)
  267. if len(result) != {num_fields:d}:
  268. raise TypeError('Expected {num_fields:d} arguments, got %d' % len(result))
  269. return result
  270. def _replace(_self, **kwds):
  271. 'Return a new {typename} object replacing specified fields with new values'
  272. result = _self._make(map(kwds.pop, {field_names!r}, _self))
  273. if kwds:
  274. raise ValueError('Got unexpected field names: %r' % list(kwds))
  275. return result
  276. def __repr__(self):
  277. 'Return a nicely formatted representation string'
  278. return self.__class__.__name__ + '({repr_fmt})' % self
  279. def _asdict(self):
  280. 'Return a new OrderedDict which maps field names to their values.'
  281. return OrderedDict(zip(self._fields, self))
  282. def __getnewargs__(self):
  283. 'Return self as a plain tuple. Used by copy and pickle.'
  284. return tuple(self)
  285. {field_defs}
  286. """
  287. _repr_template = '{name}=%r'
  288. _field_template = '''\
  289. {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}')
  290. '''
  291. def namedtuple(typename, field_names, verbose=False, rename=False):
  292. """Returns a new subclass of tuple with named fields.
  293. >>> Point = namedtuple('Point', ['x', 'y'])
  294. >>> Point.__doc__ # docstring for the new class
  295. 'Point(x, y)'
  296. >>> p = Point(11, y=22) # instantiate with positional args or keywords
  297. >>> p[0] + p[1] # indexable like a plain tuple
  298. 33
  299. >>> x, y = p # unpack like a regular tuple
  300. >>> x, y
  301. (11, 22)
  302. >>> p.x + p.y # fields also accessible by name
  303. 33
  304. >>> d = p._asdict() # convert to a dictionary
  305. >>> d['x']
  306. 11
  307. >>> Point(**d) # convert from a dictionary
  308. Point(x=11, y=22)
  309. >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields
  310. Point(x=100, y=22)
  311. """
  312. # Validate the field names. At the user's option, either generate an error
  313. # message or automatically replace the field name with a valid name.
  314. if isinstance(field_names, str):
  315. field_names = field_names.replace(',', ' ').split()
  316. field_names = list(map(str, field_names))
  317. typename = str(typename)
  318. if rename:
  319. seen = set()
  320. for index, name in enumerate(field_names):
  321. if (not name.isidentifier()
  322. or _iskeyword(name)
  323. or name.startswith('_')
  324. or name in seen):
  325. field_names[index] = '_%d' % index
  326. seen.add(name)
  327. for name in [typename] + field_names:
  328. if type(name) != str:
  329. raise TypeError('Type names and field names must be strings')
  330. if not name.isidentifier():
  331. raise ValueError('Type names and field names must be valid '
  332. 'identifiers: %r' % name)
  333. if _iskeyword(name):
  334. raise ValueError('Type names and field names cannot be a '
  335. 'keyword: %r' % name)
  336. seen = set()
  337. for name in field_names:
  338. if name.startswith('_') and not rename:
  339. raise ValueError('Field names cannot start with an underscore: '
  340. '%r' % name)
  341. if name in seen:
  342. raise ValueError('Encountered duplicate field name: %r' % name)
  343. seen.add(name)
  344. # Fill-in the class template
  345. class_definition = _class_template.format(
  346. typename = typename,
  347. field_names = tuple(field_names),
  348. num_fields = len(field_names),
  349. arg_list = repr(tuple(field_names)).replace("'", "")[1:-1],
  350. repr_fmt = ', '.join(_repr_template.format(name=name)
  351. for name in field_names),
  352. field_defs = '\n'.join(_field_template.format(index=index, name=name)
  353. for index, name in enumerate(field_names))
  354. )
  355. # Execute the template string in a temporary namespace and support
  356. # tracing utilities by setting a value for frame.f_globals['__name__']
  357. namespace = dict(__name__='namedtuple_%s' % typename)
  358. exec(class_definition, namespace)
  359. result = namespace[typename]
  360. result._source = class_definition
  361. if verbose:
  362. print(result._source)
  363. # For pickling to work, the __module__ variable needs to be set to the frame
  364. # where the named tuple is created. Bypass this step in environments where
  365. # sys._getframe is not defined (Jython for example) or sys._getframe is not
  366. # defined for arguments greater than 0 (IronPython).
  367. try:
  368. result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__')
  369. except (AttributeError, ValueError):
  370. pass
  371. return result
  372. ########################################################################
  373. ### Counter
  374. ########################################################################
  375. def _count_elements(mapping, iterable):
  376. 'Tally elements from the iterable.'
  377. mapping_get = mapping.get
  378. for elem in iterable:
  379. mapping[elem] = mapping_get(elem, 0) + 1
  380. try: # Load C helper function if available
  381. from _collections import _count_elements
  382. except ImportError:
  383. pass
  384. class Counter(dict):
  385. '''Dict subclass for counting hashable items. Sometimes called a bag
  386. or multiset. Elements are stored as dictionary keys and their counts
  387. are stored as dictionary values.
  388. >>> c = Counter('abcdeabcdabcaba') # count elements from a string
  389. >>> c.most_common(3) # three most common elements
  390. [('a', 5), ('b', 4), ('c', 3)]
  391. >>> sorted(c) # list all unique elements
  392. ['a', 'b', 'c', 'd', 'e']
  393. >>> ''.join(sorted(c.elements())) # list elements with repetitions
  394. 'aaaaabbbbcccdde'
  395. >>> sum(c.values()) # total of all counts
  396. 15
  397. >>> c['a'] # count of letter 'a'
  398. 5
  399. >>> for elem in 'shazam': # update counts from an iterable
  400. ... c[elem] += 1 # by adding 1 to each element's count
  401. >>> c['a'] # now there are seven 'a'
  402. 7
  403. >>> del c['b'] # remove all 'b'
  404. >>> c['b'] # now there are zero 'b'
  405. 0
  406. >>> d = Counter('simsalabim') # make another counter
  407. >>> c.update(d) # add in the second counter
  408. >>> c['a'] # now there are nine 'a'
  409. 9
  410. >>> c.clear() # empty the counter
  411. >>> c
  412. Counter()
  413. Note: If a count is set to zero or reduced to zero, it will remain
  414. in the counter until the entry is deleted or the counter is cleared:
  415. >>> c = Counter('aaabbc')
  416. >>> c['b'] -= 2 # reduce the count of 'b' by two
  417. >>> c.most_common() # 'b' is still in, but its count is zero
  418. [('a', 3), ('c', 1), ('b', 0)]
  419. '''
  420. # References:
  421. # http://en.wikipedia.org/wiki/Multiset
  422. # http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html
  423. # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm
  424. # http://code.activestate.com/recipes/259174/
  425. # Knuth, TAOCP Vol. II section 4.6.3
  426. def __init__(*args, **kwds):
  427. '''Create a new, empty Counter object. And if given, count elements
  428. from an input iterable. Or, initialize the count from another mapping
  429. of elements to their counts.
  430. >>> c = Counter() # a new, empty counter
  431. >>> c = Counter('gallahad') # a new counter from an iterable
  432. >>> c = Counter({'a': 4, 'b': 2}) # a new counter from a mapping
  433. >>> c = Counter(a=4, b=2) # a new counter from keyword args
  434. '''
  435. if not args:
  436. raise TypeError("descriptor '__init__' of 'Counter' object "
  437. "needs an argument")
  438. self, *args = args
  439. if len(args) > 1:
  440. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  441. super(Counter, self).__init__()
  442. self.update(*args, **kwds)
  443. def __missing__(self, key):
  444. 'The count of elements not in the Counter is zero.'
  445. # Needed so that self[missing_item] does not raise KeyError
  446. return 0
  447. def most_common(self, n=None):
  448. '''List the n most common elements and their counts from the most
  449. common to the least. If n is None, then list all element counts.
  450. >>> Counter('abcdeabcdabcaba').most_common(3)
  451. [('a', 5), ('b', 4), ('c', 3)]
  452. '''
  453. # Emulate Bag.sortedByCount from Smalltalk
  454. if n is None:
  455. return sorted(self.items(), key=_itemgetter(1), reverse=True)
  456. return _heapq.nlargest(n, self.items(), key=_itemgetter(1))
  457. def elements(self):
  458. '''Iterator over elements repeating each as many times as its count.
  459. >>> c = Counter('ABCABC')
  460. >>> sorted(c.elements())
  461. ['A', 'A', 'B', 'B', 'C', 'C']
  462. # Knuth's example for prime factors of 1836: 2**2 * 3**3 * 17**1
  463. >>> prime_factors = Counter({2: 2, 3: 3, 17: 1})
  464. >>> product = 1
  465. >>> for factor in prime_factors.elements(): # loop over factors
  466. ... product *= factor # and multiply them
  467. >>> product
  468. 1836
  469. Note, if an element's count has been set to zero or is a negative
  470. number, elements() will ignore it.
  471. '''
  472. # Emulate Bag.do from Smalltalk and Multiset.begin from C++.
  473. return _chain.from_iterable(_starmap(_repeat, self.items()))
  474. # Override dict methods where necessary
  475. @classmethod
  476. def fromkeys(cls, iterable, v=None):
  477. # There is no equivalent method for counters because setting v=1
  478. # means that no element can have a count greater than one.
  479. raise NotImplementedError(
  480. 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.')
  481. def update(*args, **kwds):
  482. '''Like dict.update() but add counts instead of replacing them.
  483. Source can be an iterable, a dictionary, or another Counter instance.
  484. >>> c = Counter('which')
  485. >>> c.update('witch') # add elements from another iterable
  486. >>> d = Counter('watch')
  487. >>> c.update(d) # add elements from another counter
  488. >>> c['h'] # four 'h' in which, witch, and watch
  489. 4
  490. '''
  491. # The regular dict.update() operation makes no sense here because the
  492. # replace behavior results in the some of original untouched counts
  493. # being mixed-in with all of the other counts for a mismash that
  494. # doesn't have a straight-forward interpretation in most counting
  495. # contexts. Instead, we implement straight-addition. Both the inputs
  496. # and outputs are allowed to contain zero and negative counts.
  497. if not args:
  498. raise TypeError("descriptor 'update' of 'Counter' object "
  499. "needs an argument")
  500. self, *args = args
  501. if len(args) > 1:
  502. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  503. iterable = args[0] if args else None
  504. if iterable is not None:
  505. if isinstance(iterable, Mapping):
  506. if self:
  507. self_get = self.get
  508. for elem, count in iterable.items():
  509. self[elem] = count + self_get(elem, 0)
  510. else:
  511. super(Counter, self).update(iterable) # fast path when counter is empty
  512. else:
  513. _count_elements(self, iterable)
  514. if kwds:
  515. self.update(kwds)
  516. def subtract(*args, **kwds):
  517. '''Like dict.update() but subtracts counts instead of replacing them.
  518. Counts can be reduced below zero. Both the inputs and outputs are
  519. allowed to contain zero and negative counts.
  520. Source can be an iterable, a dictionary, or another Counter instance.
  521. >>> c = Counter('which')
  522. >>> c.subtract('witch') # subtract elements from another iterable
  523. >>> c.subtract(Counter('watch')) # subtract elements from another counter
  524. >>> c['h'] # 2 in which, minus 1 in witch, minus 1 in watch
  525. 0
  526. >>> c['w'] # 1 in which, minus 1 in witch, minus 1 in watch
  527. -1
  528. '''
  529. if not args:
  530. raise TypeError("descriptor 'subtract' of 'Counter' object "
  531. "needs an argument")
  532. self, *args = args
  533. if len(args) > 1:
  534. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  535. iterable = args[0] if args else None
  536. if iterable is not None:
  537. self_get = self.get
  538. if isinstance(iterable, Mapping):
  539. for elem, count in iterable.items():
  540. self[elem] = self_get(elem, 0) - count
  541. else:
  542. for elem in iterable:
  543. self[elem] = self_get(elem, 0) - 1
  544. if kwds:
  545. self.subtract(kwds)
  546. def copy(self):
  547. 'Return a shallow copy.'
  548. return self.__class__(self)
  549. def __reduce__(self):
  550. return self.__class__, (dict(self),)
  551. def __delitem__(self, elem):
  552. 'Like dict.__delitem__() but does not raise KeyError for missing values.'
  553. if elem in self:
  554. super().__delitem__(elem)
  555. def __repr__(self):
  556. if not self:
  557. return '%s()' % self.__class__.__name__
  558. try:
  559. items = ', '.join(map('%r: %r'.__mod__, self.most_common()))
  560. return '%s({%s})' % (self.__class__.__name__, items)
  561. except TypeError:
  562. # handle case where values are not orderable
  563. return '{0}({1!r})'.format(self.__class__.__name__, dict(self))
  564. # Multiset-style mathematical operations discussed in:
  565. # Knuth TAOCP Volume II section 4.6.3 exercise 19
  566. # and at http://en.wikipedia.org/wiki/Multiset
  567. #
  568. # Outputs guaranteed to only include positive counts.
  569. #
  570. # To strip negative and zero counts, add-in an empty counter:
  571. # c += Counter()
  572. def __add__(self, other):
  573. '''Add counts from two counters.
  574. >>> Counter('abbb') + Counter('bcc')
  575. Counter({'b': 4, 'c': 2, 'a': 1})
  576. '''
  577. if not isinstance(other, Counter):
  578. return NotImplemented
  579. result = Counter()
  580. for elem, count in self.items():
  581. newcount = count + other[elem]
  582. if newcount > 0:
  583. result[elem] = newcount
  584. for elem, count in other.items():
  585. if elem not in self and count > 0:
  586. result[elem] = count
  587. return result
  588. def __sub__(self, other):
  589. ''' Subtract count, but keep only results with positive counts.
  590. >>> Counter('abbbc') - Counter('bccd')
  591. Counter({'b': 2, 'a': 1})
  592. '''
  593. if not isinstance(other, Counter):
  594. return NotImplemented
  595. result = Counter()
  596. for elem, count in self.items():
  597. newcount = count - other[elem]
  598. if newcount > 0:
  599. result[elem] = newcount
  600. for elem, count in other.items():
  601. if elem not in self and count < 0:
  602. result[elem] = 0 - count
  603. return result
  604. def __or__(self, other):
  605. '''Union is the maximum of value in either of the input counters.
  606. >>> Counter('abbb') | Counter('bcc')
  607. Counter({'b': 3, 'c': 2, 'a': 1})
  608. '''
  609. if not isinstance(other, Counter):
  610. return NotImplemented
  611. result = Counter()
  612. for elem, count in self.items():
  613. other_count = other[elem]
  614. newcount = other_count if count < other_count else count
  615. if newcount > 0:
  616. result[elem] = newcount
  617. for elem, count in other.items():
  618. if elem not in self and count > 0:
  619. result[elem] = count
  620. return result
  621. def __and__(self, other):
  622. ''' Intersection is the minimum of corresponding counts.
  623. >>> Counter('abbb') & Counter('bcc')
  624. Counter({'b': 1})
  625. '''
  626. if not isinstance(other, Counter):
  627. return NotImplemented
  628. result = Counter()
  629. for elem, count in self.items():
  630. other_count = other[elem]
  631. newcount = count if count < other_count else other_count
  632. if newcount > 0:
  633. result[elem] = newcount
  634. return result
  635. def __pos__(self):
  636. 'Adds an empty counter, effectively stripping negative and zero counts'
  637. result = Counter()
  638. for elem, count in self.items():
  639. if count > 0:
  640. result[elem] = count
  641. return result
  642. def __neg__(self):
  643. '''Subtracts from an empty counter. Strips positive and zero counts,
  644. and flips the sign on negative counts.
  645. '''
  646. result = Counter()
  647. for elem, count in self.items():
  648. if count < 0:
  649. result[elem] = 0 - count
  650. return result
  651. def _keep_positive(self):
  652. '''Internal method to strip elements with a negative or zero count'''
  653. nonpositive = [elem for elem, count in self.items() if not count > 0]
  654. for elem in nonpositive:
  655. del self[elem]
  656. return self
  657. def __iadd__(self, other):
  658. '''Inplace add from another counter, keeping only positive counts.
  659. >>> c = Counter('abbb')
  660. >>> c += Counter('bcc')
  661. >>> c
  662. Counter({'b': 4, 'c': 2, 'a': 1})
  663. '''
  664. for elem, count in other.items():
  665. self[elem] += count
  666. return self._keep_positive()
  667. def __isub__(self, other):
  668. '''Inplace subtract counter, but keep only results with positive counts.
  669. >>> c = Counter('abbbc')
  670. >>> c -= Counter('bccd')
  671. >>> c
  672. Counter({'b': 2, 'a': 1})
  673. '''
  674. for elem, count in other.items():
  675. self[elem] -= count
  676. return self._keep_positive()
  677. def __ior__(self, other):
  678. '''Inplace union is the maximum of value from either counter.
  679. >>> c = Counter('abbb')
  680. >>> c |= Counter('bcc')
  681. >>> c
  682. Counter({'b': 3, 'c': 2, 'a': 1})
  683. '''
  684. for elem, other_count in other.items():
  685. count = self[elem]
  686. if other_count > count:
  687. self[elem] = other_count
  688. return self._keep_positive()
  689. def __iand__(self, other):
  690. '''Inplace intersection is the minimum of corresponding counts.
  691. >>> c = Counter('abbb')
  692. >>> c &= Counter('bcc')
  693. >>> c
  694. Counter({'b': 1})
  695. '''
  696. for elem, count in self.items():
  697. other_count = other[elem]
  698. if other_count < count:
  699. self[elem] = other_count
  700. return self._keep_positive()
  701. ########################################################################
  702. ### ChainMap (helper for configparser and string.Template)
  703. ########################################################################
  704. class ChainMap(MutableMapping):
  705. ''' A ChainMap groups multiple dicts (or other mappings) together
  706. to create a single, updateable view.
  707. The underlying mappings are stored in a list. That list is public and can
  708. be accessed or updated using the *maps* attribute. There is no other
  709. state.
  710. Lookups search the underlying mappings successively until a key is found.
  711. In contrast, writes, updates, and deletions only operate on the first
  712. mapping.
  713. '''
  714. def __init__(self, *maps):
  715. '''Initialize a ChainMap by setting *maps* to the given mappings.
  716. If no mappings are provided, a single empty dictionary is used.
  717. '''
  718. self.maps = list(maps) or [{}] # always at least one map
  719. def __missing__(self, key):
  720. raise KeyError(key)
  721. def __getitem__(self, key):
  722. for mapping in self.maps:
  723. try:
  724. return mapping[key] # can't use 'key in mapping' with defaultdict
  725. except KeyError:
  726. pass
  727. return self.__missing__(key) # support subclasses that define __missing__
  728. def get(self, key, default=None):
  729. return self[key] if key in self else default
  730. def __len__(self):
  731. return len(set().union(*self.maps)) # reuses stored hash values if possible
  732. def __iter__(self):
  733. return iter(set().union(*self.maps))
  734. def __contains__(self, key):
  735. return any(key in m for m in self.maps)
  736. def __bool__(self):
  737. return any(self.maps)
  738. @_recursive_repr()
  739. def __repr__(self):
  740. return '{0.__class__.__name__}({1})'.format(
  741. self, ', '.join(map(repr, self.maps)))
  742. @classmethod
  743. def fromkeys(cls, iterable, *args):
  744. 'Create a ChainMap with a single dict created from the iterable.'
  745. return cls(dict.fromkeys(iterable, *args))
  746. def copy(self):
  747. 'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
  748. return self.__class__(self.maps[0].copy(), *self.maps[1:])
  749. __copy__ = copy
  750. def new_child(self, m=None): # like Django's Context.push()
  751. '''New ChainMap with a new map followed by all previous maps.
  752. If no map is provided, an empty dict is used.
  753. '''
  754. if m is None:
  755. m = {}
  756. return self.__class__(m, *self.maps)
  757. @property
  758. def parents(self): # like Django's Context.pop()
  759. 'New ChainMap from maps[1:].'
  760. return self.__class__(*self.maps[1:])
  761. def __setitem__(self, key, value):
  762. self.maps[0][key] = value
  763. def __delitem__(self, key):
  764. try:
  765. del self.maps[0][key]
  766. except KeyError:
  767. raise KeyError('Key not found in the first mapping: {!r}'.format(key))
  768. def popitem(self):
  769. 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.'
  770. try:
  771. return self.maps[0].popitem()
  772. except KeyError:
  773. raise KeyError('No keys found in the first mapping.')
  774. def pop(self, key, *args):
  775. 'Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0].'
  776. try:
  777. return self.maps[0].pop(key, *args)
  778. except KeyError:
  779. raise KeyError('Key not found in the first mapping: {!r}'.format(key))
  780. def clear(self):
  781. 'Clear maps[0], leaving maps[1:] intact.'
  782. self.maps[0].clear()
  783. ################################################################################
  784. ### UserDict
  785. ################################################################################
  786. class UserDict(MutableMapping):
  787. # Start by filling-out the abstract methods
  788. def __init__(*args, **kwargs):
  789. if not args:
  790. raise TypeError("descriptor '__init__' of 'UserDict' object "
  791. "needs an argument")
  792. self, *args = args
  793. if len(args) > 1:
  794. raise TypeError('expected at most 1 arguments, got %d' % len(args))
  795. if args:
  796. dict = args[0]
  797. elif 'dict' in kwargs:
  798. dict = kwargs.pop('dict')
  799. import warnings
  800. warnings.warn("Passing 'dict' as keyword argument is deprecated",
  801. PendingDeprecationWarning, stacklevel=2)
  802. else:
  803. dict = None
  804. self.data = {}
  805. if dict is not None:
  806. self.update(dict)
  807. if len(kwargs):
  808. self.update(kwargs)
  809. def __len__(self): return len(self.data)
  810. def __getitem__(self, key):
  811. if key in self.data:
  812. return self.data[key]
  813. if hasattr(self.__class__, "__missing__"):
  814. return self.__class__.__missing__(self, key)
  815. raise KeyError(key)
  816. def __setitem__(self, key, item): self.data[key] = item
  817. def __delitem__(self, key): del self.data[key]
  818. def __iter__(self):
  819. return iter(self.data)
  820. # Modify __contains__ to work correctly when __missing__ is present
  821. def __contains__(self, key):
  822. return key in self.data
  823. # Now, add the methods in dicts but not in MutableMapping
  824. def __repr__(self): return repr(self.data)
  825. def copy(self):
  826. if self.__class__ is UserDict:
  827. return UserDict(self.data.copy())
  828. import copy
  829. data = self.data
  830. try:
  831. self.data = {}
  832. c = copy.copy(self)
  833. finally:
  834. self.data = data
  835. c.update(self)
  836. return c
  837. @classmethod
  838. def fromkeys(cls, iterable, value=None):
  839. d = cls()
  840. for key in iterable:
  841. d[key] = value
  842. return d
  843. ################################################################################
  844. ### UserList
  845. ################################################################################
  846. class UserList(MutableSequence):
  847. """A more or less complete user-defined wrapper around list objects."""
  848. def __init__(self, initlist=None):
  849. self.data = []
  850. if initlist is not None:
  851. # XXX should this accept an arbitrary sequence?
  852. if type(initlist) == type(self.data):
  853. self.data[:] = initlist
  854. elif isinstance(initlist, UserList):
  855. self.data[:] = initlist.data[:]
  856. else:
  857. self.data = list(initlist)
  858. def __repr__(self): return repr(self.data)
  859. def __lt__(self, other): return self.data < self.__cast(other)
  860. def __le__(self, other): return self.data <= self.__cast(other)
  861. def __eq__(self, other): return self.data == self.__cast(other)
  862. def __gt__(self, other): return self.data > self.__cast(other)
  863. def __ge__(self, other): return self.data >= self.__cast(other)
  864. def __cast(self, other):
  865. return other.data if isinstance(other, UserList) else other
  866. def __contains__(self, item): return item in self.data
  867. def __len__(self): return len(self.data)
  868. def __getitem__(self, i): return self.data[i]
  869. def __setitem__(self, i, item): self.data[i] = item
  870. def __delitem__(self, i): del self.data[i]
  871. def __add__(self, other):
  872. if isinstance(other, UserList):
  873. return self.__class__(self.data + other.data)
  874. elif isinstance(other, type(self.data)):
  875. return self.__class__(self.data + other)
  876. return self.__class__(self.data + list(other))
  877. def __radd__(self, other):
  878. if isinstance(other, UserList):
  879. return self.__class__(other.data + self.data)
  880. elif isinstance(other, type(self.data)):
  881. return self.__class__(other + self.data)
  882. return self.__class__(list(other) + self.data)
  883. def __iadd__(self, other):
  884. if isinstance(other, UserList):
  885. self.data += other.data
  886. elif isinstance(other, type(self.data)):
  887. self.data += other
  888. else:
  889. self.data += list(other)
  890. return self
  891. def __mul__(self, n):
  892. return self.__class__(self.data*n)
  893. __rmul__ = __mul__
  894. def __imul__(self, n):
  895. self.data *= n
  896. return self
  897. def append(self, item): self.data.append(item)
  898. def insert(self, i, item): self.data.insert(i, item)
  899. def pop(self, i=-1): return self.data.pop(i)
  900. def remove(self, item): self.data.remove(item)
  901. def clear(self): self.data.clear()
  902. def copy(self): return self.__class__(self)
  903. def count(self, item): return self.data.count(item)
  904. def index(self, item, *args): return self.data.index(item, *args)
  905. def reverse(self): self.data.reverse()
  906. def sort(self, *args, **kwds): self.data.sort(*args, **kwds)
  907. def extend(self, other):
  908. if isinstance(other, UserList):
  909. self.data.extend(other.data)
  910. else:
  911. self.data.extend(other)
  912. ################################################################################
  913. ### UserString
  914. ################################################################################
  915. class UserString(Sequence):
  916. def __init__(self, seq):
  917. if isinstance(seq, str):
  918. self.data = seq
  919. elif isinstance(seq, UserString):
  920. self.data = seq.data[:]
  921. else:
  922. self.data = str(seq)
  923. def __str__(self): return str(self.data)
  924. def __repr__(self): return repr(self.data)
  925. def __int__(self): return int(self.data)
  926. def __float__(self): return float(self.data)
  927. def __complex__(self): return complex(self.data)
  928. def __hash__(self): return hash(self.data)
  929. def __getnewargs__(self):
  930. return (self.data[:],)
  931. def __eq__(self, string):
  932. if isinstance(string, UserString):
  933. return self.data == string.data
  934. return self.data == string
  935. def __lt__(self, string):
  936. if isinstance(string, UserString):
  937. return self.data < string.data
  938. return self.data < string
  939. def __le__(self, string):
  940. if isinstance(string, UserString):
  941. return self.data <= string.data
  942. return self.data <= string
  943. def __gt__(self, string):
  944. if isinstance(string, UserString):
  945. return self.data > string.data
  946. return self.data > string
  947. def __ge__(self, string):
  948. if isinstance(string, UserString):
  949. return self.data >= string.data
  950. return self.data >= string
  951. def __contains__(self, char):
  952. if isinstance(char, UserString):
  953. char = char.data
  954. return char in self.data
  955. def __len__(self): return len(self.data)
  956. def __getitem__(self, index): return self.__class__(self.data[index])
  957. def __add__(self, other):
  958. if isinstance(other, UserString):
  959. return self.__class__(self.data + other.data)
  960. elif isinstance(other, str):
  961. return self.__class__(self.data + other)
  962. return self.__class__(self.data + str(other))
  963. def __radd__(self, other):
  964. if isinstance(other, str):
  965. return self.__class__(other + self.data)
  966. return self.__class__(str(other) + self.data)
  967. def __mul__(self, n):
  968. return self.__class__(self.data*n)
  969. __rmul__ = __mul__
  970. def __mod__(self, args):
  971. return self.__class__(self.data % args)
  972. def __rmod__(self, format):
  973. return self.__class__(format % args)
  974. # the following methods are defined in alphabetical order:
  975. def capitalize(self): return self.__class__(self.data.capitalize())
  976. def casefold(self):
  977. return self.__class__(self.data.casefold())
  978. def center(self, width, *args):
  979. return self.__class__(self.data.center(width, *args))
  980. def count(self, sub, start=0, end=_sys.maxsize):
  981. if isinstance(sub, UserString):
  982. sub = sub.data
  983. return self.data.count(sub, start, end)
  984. def encode(self, encoding=None, errors=None): # XXX improve this?
  985. if encoding:
  986. if errors:
  987. return self.__class__(self.data.encode(encoding, errors))
  988. return self.__class__(self.data.encode(encoding))
  989. return self.__class__(self.data.encode())
  990. def endswith(self, suffix, start=0, end=_sys.maxsize):
  991. return self.data.endswith(suffix, start, end)
  992. def expandtabs(self, tabsize=8):
  993. return self.__class__(self.data.expandtabs(tabsize))
  994. def find(self, sub, start=0, end=_sys.maxsize):
  995. if isinstance(sub, UserString):
  996. sub = sub.data
  997. return self.data.find(sub, start, end)
  998. def format(self, *args, **kwds):
  999. return self.data.format(*args, **kwds)
  1000. def format_map(self, mapping):
  1001. return self.data.format_map(mapping)
  1002. def index(self, sub, start=0, end=_sys.maxsize):
  1003. return self.data.index(sub, start, end)
  1004. def isalpha(self): return self.data.isalpha()
  1005. def isalnum(self): return self.data.isalnum()
  1006. def isdecimal(self): return self.data.isdecimal()
  1007. def isdigit(self): return self.data.isdigit()
  1008. def isidentifier(self): return self.data.isidentifier()
  1009. def islower(self): return self.data.islower()
  1010. def isnumeric(self): return self.data.isnumeric()
  1011. def isprintable(self): return self.data.isprintable()
  1012. def isspace(self): return self.data.isspace()
  1013. def istitle(self): return self.data.istitle()
  1014. def isupper(self): return self.data.isupper()
  1015. def join(self, seq): return self.data.join(seq)
  1016. def ljust(self, width, *args):
  1017. return self.__class__(self.data.ljust(width, *args))
  1018. def lower(self): return self.__class__(self.data.lower())
  1019. def lstrip(self, chars=None): return self.__class__(self.data.lstrip(chars))
  1020. maketrans = str.maketrans
  1021. def partition(self, sep):
  1022. return self.data.partition(sep)
  1023. def replace(self, old, new, maxsplit=-1):
  1024. if isinstance(old, UserString):
  1025. old = old.data
  1026. if isinstance(new, UserString):
  1027. new = new.data
  1028. return self.__class__(self.data.replace(old, new, maxsplit))
  1029. def rfind(self, sub, start=0, end=_sys.maxsize):
  1030. if isinstance(sub, UserString):
  1031. sub = sub.data
  1032. return self.data.rfind(sub, start, end)
  1033. def rindex(self, sub, start=0, end=_sys.maxsize):
  1034. return self.data.rindex(sub, start, end)
  1035. def rjust(self, width, *args):
  1036. return self.__class__(self.data.rjust(width, *args))
  1037. def rpartition(self, sep):
  1038. return self.data.rpartition(sep)
  1039. def rstrip(self, chars=None):
  1040. return self.__class__(self.data.rstrip(chars))
  1041. def split(self, sep=None, maxsplit=-1):
  1042. return self.data.split(sep, maxsplit)
  1043. def rsplit(self, sep=None, maxsplit=-1):
  1044. return self.data.rsplit(sep, maxsplit)
  1045. def splitlines(self, keepends=False): return self.data.splitlines(keepends)
  1046. def startswith(self, prefix, start=0, end=_sys.maxsize):
  1047. return self.data.startswith(prefix, start, end)
  1048. def strip(self, chars=None): return self.__class__(self.data.strip(chars))
  1049. def swapcase(self): return self.__class__(self.data.swapcase())
  1050. def title(self): return self.__class__(self.data.title())
  1051. def translate(self, *args):
  1052. return self.__class__(self.data.translate(*args))
  1053. def upper(self): return self.__class__(self.data.upper())
  1054. def zfill(self, width): return self.__class__(self.data.zfill(width))