PMurHash.c 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. /*-----------------------------------------------------------------------------
  2. * MurmurHash3 was written by Austin Appleby, and is placed in the public
  3. * domain.
  4. *
  5. * This implementation was written by Shane Day, and is also public domain.
  6. *
  7. * This is a portable ANSI C implementation of MurmurHash3_x86_32 (Murmur3A)
  8. * with support for progressive processing.
  9. */
  10. /*-----------------------------------------------------------------------------
  11. If you want to understand the MurmurHash algorithm you would be much better
  12. off reading the original source. Just point your browser at:
  13. http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
  14. What this version provides?
  15. 1. Progressive data feeding. Useful when the entire payload to be hashed
  16. does not fit in memory or when the data is streamed through the application.
  17. Also useful when hashing a number of strings with a common prefix. A partial
  18. hash of a prefix string can be generated and reused for each suffix string.
  19. How does it work?
  20. We can only process entire 32 bit chunks of input, except for the very end
  21. that may be shorter. So along with the partial hash we need to give back to
  22. the caller a carry containing up to 3 bytes that we were unable to process.
  23. This carry also needs to record the number of bytes the carry holds. I use
  24. the low 2 bits as a count (0..3) and the carry bytes are shifted into the
  25. high byte in stream order.
  26. To handle endianess I simply use a macro that reads a uint32_t and define
  27. that macro to be a direct read on little endian machines, a read and swap
  28. on big endian machines, or a byte-by-byte read if the endianess is unknown.
  29. -----------------------------------------------------------------------------*/
  30. #include "PMurHash.h"
  31. // /* MSVC warnings we choose to ignore */
  32. // #if defined(_MSC_VER)
  33. // #pragma warning(disable: 4127) /* conditional expression is constant */
  34. // #endif
  35. /*-----------------------------------------------------------------------------
  36. * Endianess, misalignment capabilities and util macros
  37. *
  38. * The following 3 macros are defined in this section. The other macros defined
  39. * are only needed to help derive these 3.
  40. *
  41. * READ_UINT32(x) Read a little endian unsigned 32-bit int
  42. * UNALIGNED_SAFE Defined if READ_UINT32 works on non-word boundaries
  43. * ROTL32(x,r) Rotate x left by r bits
  44. */
  45. /* I386 or AMD64 */
  46. #if defined(_M_I86) || defined(_M_IX86) || defined(_X86_) || defined(__i386__) || defined(__i386) || defined(i386) \
  47. || defined(_M_X64) || defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64)
  48. #define UNALIGNED_SAFE
  49. #endif
  50. /* I386 or AMD64 */
  51. #if defined(_M_I86) || defined(_M_IX86) || defined(_X86_) || defined(__i386__) || defined(__i386) || defined(i386) \
  52. || defined(_M_X64) || defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64)
  53. #define UNALIGNED_SAFE
  54. #endif
  55. /* Find best way to ROTL */
  56. #if defined(_MSC_VER)
  57. #define FORCE_INLINE static __forceinline
  58. #include <stdlib.h> /* Microsoft put _rotl declaration in here */
  59. #define ROTL32(x,y) _rotl(x,y)
  60. #else
  61. #define FORCE_INLINE static inline __attribute__((always_inline))
  62. /* gcc recognises this code and generates a rotate instruction for CPUs with one */
  63. #define ROTL32(x,r) (((uint32_t)x << r) | ((uint32_t)x >> (32 - r)))
  64. #endif
  65. #include "endianness.h"
  66. #define READ_UINT32(ptr) getblock32((uint32_t *)ptr, 0)
  67. /*-----------------------------------------------------------------------------
  68. * Core murmurhash algorithm macros */
  69. static const uint32_t kC1 = 0xcc9e2d51;
  70. static const uint32_t kC2 = 0x1b873593;
  71. /* This is the main processing body of the algorithm. It operates
  72. * on each full 32-bits of input. */
  73. #define doblock(h1, k1) \
  74. do {\
  75. k1 *= kC1;\
  76. k1 = ROTL32(k1,15);\
  77. k1 *= kC2;\
  78. \
  79. h1 ^= k1;\
  80. h1 = ROTL32(h1,13);\
  81. h1 = h1*5+0xe6546b64;\
  82. } while(0)
  83. /* Append unaligned bytes to carry, forcing hash churn if we have 4 bytes */
  84. /* cnt=bytes to process, h1=name of h1 var, c=carry, n=bytes in c, ptr/len=payload */
  85. #define dobytes(cnt, h1, c, n, ptr, len) \
  86. do {\
  87. unsigned __cnt = cnt;\
  88. while(__cnt--) {\
  89. c = c>>8 | (uint32_t)*ptr++<<24;\
  90. n++; len--;\
  91. if(n==4) {\
  92. doblock(h1, c);\
  93. n = 0;\
  94. }\
  95. }\
  96. } while(0)
  97. /*---------------------------------------------------------------------------*/
  98. /* Main hashing function. Initialise carry to 0 and h1 to 0 or an initial seed
  99. * if wanted. Both ph1 and pcarry are required arguments. */
  100. void PMurHash32_Process(uint32_t *ph1, uint32_t *pcarry, const void *key, int len)
  101. {
  102. uint32_t h1 = *ph1;
  103. uint32_t c = *pcarry;
  104. const uint8_t *ptr = (uint8_t*)key;
  105. const uint8_t *end;
  106. /* Extract carry count from low 2 bits of c value */
  107. int n = c & 3;
  108. #if defined(UNALIGNED_SAFE)
  109. /* This CPU handles unaligned word access */
  110. // #pragma message ( "UNALIGNED_SAFE" )
  111. /* Consume any carry bytes */
  112. int i = (4-n) & 3;
  113. if(i && i <= len) {
  114. dobytes(i, h1, c, n, ptr, len);
  115. }
  116. /* Process 32-bit chunks */
  117. end = ptr + (len & ~3);
  118. for( ; ptr < end ; ptr+=4) {
  119. uint32_t k1 = READ_UINT32(ptr);
  120. doblock(h1, k1);
  121. }
  122. #else /*UNALIGNED_SAFE*/
  123. /* This CPU does not handle unaligned word access */
  124. // #pragma message ( "ALIGNED" )
  125. /* Consume enough so that the next data byte is word aligned */
  126. int i = -(intptr_t)(void *)ptr & 3;
  127. if(i && i <= len) {
  128. dobytes(i, h1, c, n, ptr, len);
  129. }
  130. /* We're now aligned. Process in aligned blocks. Specialise for each possible carry count */
  131. end = ptr + (len & ~3);
  132. switch(n) { /* how many bytes in c */
  133. case 0: /* c=[----] w=[3210] b=[3210]=w c'=[----] */
  134. for( ; ptr < end ; ptr+=4) {
  135. uint32_t k1 = READ_UINT32(ptr);
  136. doblock(h1, k1);
  137. }
  138. break;
  139. case 1: /* c=[0---] w=[4321] b=[3210]=c>>24|w<<8 c'=[4---] */
  140. for( ; ptr < end ; ptr+=4) {
  141. uint32_t k1 = c>>24;
  142. c = READ_UINT32(ptr);
  143. k1 |= c<<8;
  144. doblock(h1, k1);
  145. }
  146. break;
  147. case 2: /* c=[10--] w=[5432] b=[3210]=c>>16|w<<16 c'=[54--] */
  148. for( ; ptr < end ; ptr+=4) {
  149. uint32_t k1 = c>>16;
  150. c = READ_UINT32(ptr);
  151. k1 |= c<<16;
  152. doblock(h1, k1);
  153. }
  154. break;
  155. case 3: /* c=[210-] w=[6543] b=[3210]=c>>8|w<<24 c'=[654-] */
  156. for( ; ptr < end ; ptr+=4) {
  157. uint32_t k1 = c>>8;
  158. c = READ_UINT32(ptr);
  159. k1 |= c<<24;
  160. doblock(h1, k1);
  161. }
  162. }
  163. #endif /*UNALIGNED_SAFE*/
  164. /* Advance over whole 32-bit chunks, possibly leaving 1..3 bytes */
  165. len -= len & ~3;
  166. /* Append any remaining bytes into carry */
  167. dobytes(len, h1, c, n, ptr, len);
  168. /* Copy out new running hash and carry */
  169. *ph1 = h1;
  170. *pcarry = (c & ~0xff) | n;
  171. }
  172. /*---------------------------------------------------------------------------*/
  173. /* Finalize a hash. To match the original Murmur3A the total_length must be provided */
  174. uint32_t PMurHash32_Result(uint32_t h, uint32_t carry, uint32_t total_length)
  175. {
  176. uint32_t k1;
  177. int n = carry & 3;
  178. if(n) {
  179. k1 = carry >> (4-n)*8;
  180. k1 *= kC1; k1 = ROTL32(k1,15); k1 *= kC2; h ^= k1;
  181. }
  182. h ^= total_length;
  183. /* fmix */
  184. h ^= h >> 16;
  185. h *= 0x85ebca6b;
  186. h ^= h >> 13;
  187. h *= 0xc2b2ae35;
  188. h ^= h >> 16;
  189. return h;
  190. }