base64.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601
  1. #! /usr/bin/env python3
  2. """Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
  3. # Modified 04-Oct-1995 by Jack Jansen to use binascii module
  4. # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
  5. # Modified 22-May-2007 by Guido van Rossum to use bytes everywhere
  6. import re
  7. import struct
  8. import binascii
  9. __all__ = [
  10. # Legacy interface exports traditional RFC 2045 Base64 encodings
  11. 'encode', 'decode', 'encodebytes', 'decodebytes',
  12. # Generalized interface for other encodings
  13. 'b64encode', 'b64decode', 'b32encode', 'b32decode',
  14. 'b16encode', 'b16decode',
  15. # Base85 and Ascii85 encodings
  16. 'b85encode', 'b85decode', 'a85encode', 'a85decode',
  17. # Standard Base64 encoding
  18. 'standard_b64encode', 'standard_b64decode',
  19. # Some common Base64 alternatives. As referenced by RFC 3458, see thread
  20. # starting at:
  21. #
  22. # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html
  23. 'urlsafe_b64encode', 'urlsafe_b64decode',
  24. ]
  25. bytes_types = (bytes, bytearray) # Types acceptable as binary data
  26. def _bytes_from_decode_data(s):
  27. if isinstance(s, str):
  28. try:
  29. return s.encode('ascii')
  30. except UnicodeEncodeError:
  31. raise ValueError('string argument should contain only ASCII characters')
  32. if isinstance(s, bytes_types):
  33. return s
  34. try:
  35. return memoryview(s).tobytes()
  36. except TypeError:
  37. raise TypeError("argument should be a bytes-like object or ASCII "
  38. "string, not %r" % s.__class__.__name__) from None
  39. # Base64 encoding/decoding uses binascii
  40. def b64encode(s, altchars=None):
  41. """Encode the bytes-like object s using Base64 and return a bytes object.
  42. Optional altchars should be a byte string of length 2 which specifies an
  43. alternative alphabet for the '+' and '/' characters. This allows an
  44. application to e.g. generate url or filesystem safe Base64 strings.
  45. """
  46. # Strip off the trailing newline
  47. encoded = binascii.b2a_base64(s)[:-1]
  48. if altchars is not None:
  49. assert len(altchars) == 2, repr(altchars)
  50. return encoded.translate(bytes.maketrans(b'+/', altchars))
  51. return encoded
  52. def b64decode(s, altchars=None, validate=False):
  53. """Decode the Base64 encoded bytes-like object or ASCII string s.
  54. Optional altchars must be a bytes-like object or ASCII string of length 2
  55. which specifies the alternative alphabet used instead of the '+' and '/'
  56. characters.
  57. The result is returned as a bytes object. A binascii.Error is raised if
  58. s is incorrectly padded.
  59. If validate is False (the default), characters that are neither in the
  60. normal base-64 alphabet nor the alternative alphabet are discarded prior
  61. to the padding check. If validate is True, these non-alphabet characters
  62. in the input result in a binascii.Error.
  63. """
  64. s = _bytes_from_decode_data(s)
  65. if altchars is not None:
  66. altchars = _bytes_from_decode_data(altchars)
  67. assert len(altchars) == 2, repr(altchars)
  68. s = s.translate(bytes.maketrans(altchars, b'+/'))
  69. if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s):
  70. raise binascii.Error('Non-base64 digit found')
  71. return binascii.a2b_base64(s)
  72. def standard_b64encode(s):
  73. """Encode bytes-like object s using the standard Base64 alphabet.
  74. The result is returned as a bytes object.
  75. """
  76. return b64encode(s)
  77. def standard_b64decode(s):
  78. """Decode bytes encoded with the standard Base64 alphabet.
  79. Argument s is a bytes-like object or ASCII string to decode. The result
  80. is returned as a bytes object. A binascii.Error is raised if the input
  81. is incorrectly padded. Characters that are not in the standard alphabet
  82. are discarded prior to the padding check.
  83. """
  84. return b64decode(s)
  85. _urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_')
  86. _urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/')
  87. def urlsafe_b64encode(s):
  88. """Encode bytes using the URL- and filesystem-safe Base64 alphabet.
  89. Argument s is a bytes-like object to encode. The result is returned as a
  90. bytes object. The alphabet uses '-' instead of '+' and '_' instead of
  91. '/'.
  92. """
  93. return b64encode(s).translate(_urlsafe_encode_translation)
  94. def urlsafe_b64decode(s):
  95. """Decode bytes using the URL- and filesystem-safe Base64 alphabet.
  96. Argument s is a bytes-like object or ASCII string to decode. The result
  97. is returned as a bytes object. A binascii.Error is raised if the input
  98. is incorrectly padded. Characters that are not in the URL-safe base-64
  99. alphabet, and are not a plus '+' or slash '/', are discarded prior to the
  100. padding check.
  101. The alphabet uses '-' instead of '+' and '_' instead of '/'.
  102. """
  103. s = _bytes_from_decode_data(s)
  104. s = s.translate(_urlsafe_decode_translation)
  105. return b64decode(s)
  106. # Base32 encoding/decoding must be done in Python
  107. _b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
  108. _b32tab2 = None
  109. _b32rev = None
  110. def b32encode(s):
  111. """Encode the bytes-like object s using Base32 and return a bytes object.
  112. """
  113. global _b32tab2
  114. # Delay the initialization of the table to not waste memory
  115. # if the function is never called
  116. if _b32tab2 is None:
  117. b32tab = [bytes((i,)) for i in _b32alphabet]
  118. _b32tab2 = [a + b for a in b32tab for b in b32tab]
  119. b32tab = None
  120. if not isinstance(s, bytes_types):
  121. s = memoryview(s).tobytes()
  122. leftover = len(s) % 5
  123. # Pad the last quantum with zero bits if necessary
  124. if leftover:
  125. s = s + bytes(5 - leftover) # Don't use += !
  126. encoded = bytearray()
  127. from_bytes = int.from_bytes
  128. b32tab2 = _b32tab2
  129. for i in range(0, len(s), 5):
  130. c = from_bytes(s[i: i + 5], 'big')
  131. encoded += (b32tab2[c >> 30] + # bits 1 - 10
  132. b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20
  133. b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30
  134. b32tab2[c & 0x3ff] # bits 31 - 40
  135. )
  136. # Adjust for any leftover partial quanta
  137. if leftover == 1:
  138. encoded[-6:] = b'======'
  139. elif leftover == 2:
  140. encoded[-4:] = b'===='
  141. elif leftover == 3:
  142. encoded[-3:] = b'==='
  143. elif leftover == 4:
  144. encoded[-1:] = b'='
  145. return bytes(encoded)
  146. def b32decode(s, casefold=False, map01=None):
  147. """Decode the Base32 encoded bytes-like object or ASCII string s.
  148. Optional casefold is a flag specifying whether a lowercase alphabet is
  149. acceptable as input. For security purposes, the default is False.
  150. RFC 3548 allows for optional mapping of the digit 0 (zero) to the
  151. letter O (oh), and for optional mapping of the digit 1 (one) to
  152. either the letter I (eye) or letter L (el). The optional argument
  153. map01 when not None, specifies which letter the digit 1 should be
  154. mapped to (when map01 is not None, the digit 0 is always mapped to
  155. the letter O). For security purposes the default is None, so that
  156. 0 and 1 are not allowed in the input.
  157. The result is returned as a bytes object. A binascii.Error is raised if
  158. the input is incorrectly padded or if there are non-alphabet
  159. characters present in the input.
  160. """
  161. global _b32rev
  162. # Delay the initialization of the table to not waste memory
  163. # if the function is never called
  164. if _b32rev is None:
  165. _b32rev = {v: k for k, v in enumerate(_b32alphabet)}
  166. s = _bytes_from_decode_data(s)
  167. if len(s) % 8:
  168. raise binascii.Error('Incorrect padding')
  169. # Handle section 2.4 zero and one mapping. The flag map01 will be either
  170. # False, or the character to map the digit 1 (one) to. It should be
  171. # either L (el) or I (eye).
  172. if map01 is not None:
  173. map01 = _bytes_from_decode_data(map01)
  174. assert len(map01) == 1, repr(map01)
  175. s = s.translate(bytes.maketrans(b'01', b'O' + map01))
  176. if casefold:
  177. s = s.upper()
  178. # Strip off pad characters from the right. We need to count the pad
  179. # characters because this will tell us how many null bytes to remove from
  180. # the end of the decoded string.
  181. l = len(s)
  182. s = s.rstrip(b'=')
  183. padchars = l - len(s)
  184. # Now decode the full quanta
  185. decoded = bytearray()
  186. b32rev = _b32rev
  187. for i in range(0, len(s), 8):
  188. quanta = s[i: i + 8]
  189. acc = 0
  190. try:
  191. for c in quanta:
  192. acc = (acc << 5) + b32rev[c]
  193. except KeyError:
  194. raise binascii.Error('Non-base32 digit found') from None
  195. decoded += acc.to_bytes(5, 'big')
  196. # Process the last, partial quanta
  197. if padchars:
  198. acc <<= 5 * padchars
  199. last = acc.to_bytes(5, 'big')
  200. if padchars == 1:
  201. decoded[-5:] = last[:-1]
  202. elif padchars == 3:
  203. decoded[-5:] = last[:-2]
  204. elif padchars == 4:
  205. decoded[-5:] = last[:-3]
  206. elif padchars == 6:
  207. decoded[-5:] = last[:-4]
  208. else:
  209. raise binascii.Error('Incorrect padding')
  210. return bytes(decoded)
  211. # RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns
  212. # lowercase. The RFC also recommends against accepting input case
  213. # insensitively.
  214. def b16encode(s):
  215. """Encode the bytes-like object s using Base16 and return a bytes object.
  216. """
  217. return binascii.hexlify(s).upper()
  218. def b16decode(s, casefold=False):
  219. """Decode the Base16 encoded bytes-like object or ASCII string s.
  220. Optional casefold is a flag specifying whether a lowercase alphabet is
  221. acceptable as input. For security purposes, the default is False.
  222. The result is returned as a bytes object. A binascii.Error is raised if
  223. s is incorrectly padded or if there are non-alphabet characters present
  224. in the input.
  225. """
  226. s = _bytes_from_decode_data(s)
  227. if casefold:
  228. s = s.upper()
  229. if re.search(b'[^0-9A-F]', s):
  230. raise binascii.Error('Non-base16 digit found')
  231. return binascii.unhexlify(s)
  232. #
  233. # Ascii85 encoding/decoding
  234. #
  235. _a85chars = None
  236. _a85chars2 = None
  237. _A85START = b"<~"
  238. _A85END = b"~>"
  239. def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False):
  240. # Helper function for a85encode and b85encode
  241. if not isinstance(b, bytes_types):
  242. b = memoryview(b).tobytes()
  243. padding = (-len(b)) % 4
  244. if padding:
  245. b = b + b'\0' * padding
  246. words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b)
  247. chunks = [b'z' if foldnuls and not word else
  248. b'y' if foldspaces and word == 0x20202020 else
  249. (chars2[word // 614125] +
  250. chars2[word // 85 % 7225] +
  251. chars[word % 85])
  252. for word in words]
  253. if padding and not pad:
  254. if chunks[-1] == b'z':
  255. chunks[-1] = chars[0] * 5
  256. chunks[-1] = chunks[-1][:-padding]
  257. return b''.join(chunks)
  258. def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False):
  259. """Encode bytes-like object b using Ascii85 and return a bytes object.
  260. foldspaces is an optional flag that uses the special short sequence 'y'
  261. instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This
  262. feature is not supported by the "standard" Adobe encoding.
  263. wrapcol controls whether the output should have newline (b'\\n') characters
  264. added to it. If this is non-zero, each output line will be at most this
  265. many characters long.
  266. pad controls whether the input is padded to a multiple of 4 before
  267. encoding. Note that the btoa implementation always pads.
  268. adobe controls whether the encoded byte sequence is framed with <~ and ~>,
  269. which is used by the Adobe implementation.
  270. """
  271. global _a85chars, _a85chars2
  272. # Delay the initialization of tables to not waste memory
  273. # if the function is never called
  274. if _a85chars is None:
  275. _a85chars = [bytes((i,)) for i in range(33, 118)]
  276. _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars]
  277. result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces)
  278. if adobe:
  279. result = _A85START + result
  280. if wrapcol:
  281. wrapcol = max(2 if adobe else 1, wrapcol)
  282. chunks = [result[i: i + wrapcol]
  283. for i in range(0, len(result), wrapcol)]
  284. if adobe:
  285. if len(chunks[-1]) + 2 > wrapcol:
  286. chunks.append(b'')
  287. result = b'\n'.join(chunks)
  288. if adobe:
  289. result += _A85END
  290. return result
  291. def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'):
  292. """Decode the Ascii85 encoded bytes-like object or ASCII string b.
  293. foldspaces is a flag that specifies whether the 'y' short sequence should be
  294. accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is
  295. not supported by the "standard" Adobe encoding.
  296. adobe controls whether the input sequence is in Adobe Ascii85 format (i.e.
  297. is framed with <~ and ~>).
  298. ignorechars should be a byte string containing characters to ignore from the
  299. input. This should only contain whitespace characters, and by default
  300. contains all whitespace characters in ASCII.
  301. The result is returned as a bytes object.
  302. """
  303. b = _bytes_from_decode_data(b)
  304. if adobe:
  305. if not b.endswith(_A85END):
  306. raise ValueError(
  307. "Ascii85 encoded byte sequences must end "
  308. "with {!r}".format(_A85END)
  309. )
  310. if b.startswith(_A85START):
  311. b = b[2:-2] # Strip off start/end markers
  312. else:
  313. b = b[:-2]
  314. #
  315. # We have to go through this stepwise, so as to ignore spaces and handle
  316. # special short sequences
  317. #
  318. packI = struct.Struct('!I').pack
  319. decoded = []
  320. decoded_append = decoded.append
  321. curr = []
  322. curr_append = curr.append
  323. curr_clear = curr.clear
  324. for x in b + b'u' * 4:
  325. if b'!'[0] <= x <= b'u'[0]:
  326. curr_append(x)
  327. if len(curr) == 5:
  328. acc = 0
  329. for x in curr:
  330. acc = 85 * acc + (x - 33)
  331. try:
  332. decoded_append(packI(acc))
  333. except struct.error:
  334. raise ValueError('Ascii85 overflow') from None
  335. curr_clear()
  336. elif x == b'z'[0]:
  337. if curr:
  338. raise ValueError('z inside Ascii85 5-tuple')
  339. decoded_append(b'\0\0\0\0')
  340. elif foldspaces and x == b'y'[0]:
  341. if curr:
  342. raise ValueError('y inside Ascii85 5-tuple')
  343. decoded_append(b'\x20\x20\x20\x20')
  344. elif x in ignorechars:
  345. # Skip whitespace
  346. continue
  347. else:
  348. raise ValueError('Non-Ascii85 digit found: %c' % x)
  349. result = b''.join(decoded)
  350. padding = 4 - len(curr)
  351. if padding:
  352. # Throw away the extra padding
  353. result = result[:-padding]
  354. return result
  355. # The following code is originally taken (with permission) from Mercurial
  356. _b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
  357. b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~")
  358. _b85chars = None
  359. _b85chars2 = None
  360. _b85dec = None
  361. def b85encode(b, pad=False):
  362. """Encode bytes-like object b in base85 format and return a bytes object.
  363. If pad is true, the input is padded with b'\\0' so its length is a multiple of
  364. 4 bytes before encoding.
  365. """
  366. global _b85chars, _b85chars2
  367. # Delay the initialization of tables to not waste memory
  368. # if the function is never called
  369. if _b85chars is None:
  370. _b85chars = [bytes((i,)) for i in _b85alphabet]
  371. _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars]
  372. return _85encode(b, _b85chars, _b85chars2, pad)
  373. def b85decode(b):
  374. """Decode the base85-encoded bytes-like object or ASCII string b
  375. The result is returned as a bytes object.
  376. """
  377. global _b85dec
  378. # Delay the initialization of tables to not waste memory
  379. # if the function is never called
  380. if _b85dec is None:
  381. _b85dec = [None] * 256
  382. for i, c in enumerate(_b85alphabet):
  383. _b85dec[c] = i
  384. b = _bytes_from_decode_data(b)
  385. padding = (-len(b)) % 5
  386. b = b + b'~' * padding
  387. out = []
  388. packI = struct.Struct('!I').pack
  389. for i in range(0, len(b), 5):
  390. chunk = b[i:i + 5]
  391. acc = 0
  392. try:
  393. for c in chunk:
  394. acc = acc * 85 + _b85dec[c]
  395. except TypeError:
  396. for j, c in enumerate(chunk):
  397. if _b85dec[c] is None:
  398. raise ValueError('bad base85 character at position %d'
  399. % (i + j)) from None
  400. raise
  401. try:
  402. out.append(packI(acc))
  403. except struct.error:
  404. raise ValueError('base85 overflow in hunk starting at byte %d'
  405. % i) from None
  406. result = b''.join(out)
  407. if padding:
  408. result = result[:-padding]
  409. return result
  410. # Legacy interface. This code could be cleaned up since I don't believe
  411. # binascii has any line length limitations. It just doesn't seem worth it
  412. # though. The files should be opened in binary mode.
  413. MAXLINESIZE = 76 # Excluding the CRLF
  414. MAXBINSIZE = (MAXLINESIZE//4)*3
  415. def encode(input, output):
  416. """Encode a file; input and output are binary files."""
  417. while True:
  418. s = input.read(MAXBINSIZE)
  419. if not s:
  420. break
  421. while len(s) < MAXBINSIZE:
  422. ns = input.read(MAXBINSIZE-len(s))
  423. if not ns:
  424. break
  425. s += ns
  426. line = binascii.b2a_base64(s)
  427. output.write(line)
  428. def decode(input, output):
  429. """Decode a file; input and output are binary files."""
  430. while True:
  431. line = input.readline()
  432. if not line:
  433. break
  434. s = binascii.a2b_base64(line)
  435. output.write(s)
  436. def _input_type_check(s):
  437. try:
  438. m = memoryview(s)
  439. except TypeError as err:
  440. msg = "expected bytes-like object, not %s" % s.__class__.__name__
  441. raise TypeError(msg) from err
  442. if m.format not in ('c', 'b', 'B'):
  443. msg = ("expected single byte elements, not %r from %s" %
  444. (m.format, s.__class__.__name__))
  445. raise TypeError(msg)
  446. if m.ndim != 1:
  447. msg = ("expected 1-D data, not %d-D data from %s" %
  448. (m.ndim, s.__class__.__name__))
  449. raise TypeError(msg)
  450. def encodebytes(s):
  451. """Encode a bytestring into a bytes object containing multiple lines
  452. of base-64 data."""
  453. _input_type_check(s)
  454. pieces = []
  455. for i in range(0, len(s), MAXBINSIZE):
  456. chunk = s[i : i + MAXBINSIZE]
  457. pieces.append(binascii.b2a_base64(chunk))
  458. return b"".join(pieces)
  459. def encodestring(s):
  460. """Legacy alias of encodebytes()."""
  461. import warnings
  462. warnings.warn("encodestring() is a deprecated alias, use encodebytes()",
  463. DeprecationWarning, 2)
  464. return encodebytes(s)
  465. def decodebytes(s):
  466. """Decode a bytestring of base-64 data into a bytes object."""
  467. _input_type_check(s)
  468. return binascii.a2b_base64(s)
  469. def decodestring(s):
  470. """Legacy alias of decodebytes()."""
  471. import warnings
  472. warnings.warn("decodestring() is a deprecated alias, use decodebytes()",
  473. DeprecationWarning, 2)
  474. return decodebytes(s)
  475. # Usable as a script...
  476. def main():
  477. """Small main program"""
  478. import sys, getopt
  479. try:
  480. opts, args = getopt.getopt(sys.argv[1:], 'deut')
  481. except getopt.error as msg:
  482. sys.stdout = sys.stderr
  483. print(msg)
  484. print("""usage: %s [-d|-e|-u|-t] [file|-]
  485. -d, -u: decode
  486. -e: encode (default)
  487. -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0])
  488. sys.exit(2)
  489. func = encode
  490. for o, a in opts:
  491. if o == '-e': func = encode
  492. if o == '-d': func = decode
  493. if o == '-u': func = decode
  494. if o == '-t': test(); return
  495. if args and args[0] != '-':
  496. with open(args[0], 'rb') as f:
  497. func(f, sys.stdout.buffer)
  498. else:
  499. func(sys.stdin.buffer, sys.stdout.buffer)
  500. def test():
  501. s0 = b"Aladdin:open sesame"
  502. print(repr(s0))
  503. s1 = encodebytes(s0)
  504. print(repr(s1))
  505. s2 = decodebytes(s1)
  506. print(repr(s2))
  507. assert s0 == s2
  508. if __name__ == '__main__':
  509. main()