fastmem.h 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153
  1. // Protocol Buffers - Google's data interchange format
  2. // Copyright 2014 Google Inc. All rights reserved.
  3. // https://developers.google.com/protocol-buffers/
  4. //
  5. // Redistribution and use in source and binary forms, with or without
  6. // modification, are permitted provided that the following conditions are
  7. // met:
  8. //
  9. // * Redistributions of source code must retain the above copyright
  10. // notice, this list of conditions and the following disclaimer.
  11. // * Redistributions in binary form must reproduce the above
  12. // copyright notice, this list of conditions and the following disclaimer
  13. // in the documentation and/or other materials provided with the
  14. // distribution.
  15. // * Neither the name of Google Inc. nor the names of its
  16. // contributors may be used to endorse or promote products derived from
  17. // this software without specific prior written permission.
  18. //
  19. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. // Fast memory copying and comparison routines.
  31. // strings::fastmemcmp_inlined() replaces memcmp()
  32. // strings::memcpy_inlined() replaces memcpy()
  33. // strings::memeq(a, b, n) replaces memcmp(a, b, n) == 0
  34. //
  35. // strings::*_inlined() routines are inline versions of the
  36. // routines exported by this module. Sometimes using the inlined
  37. // versions is faster. Measure before using the inlined versions.
  38. //
  39. // Performance measurement:
  40. // strings::fastmemcmp_inlined
  41. // Analysis: memcmp, fastmemcmp_inlined, fastmemcmp
  42. // 2012-01-30
  43. #ifndef GOOGLE_PROTOBUF_STUBS_FASTMEM_H_
  44. #define GOOGLE_PROTOBUF_STUBS_FASTMEM_H_
  45. #include <stddef.h>
  46. #include <stdio.h>
  47. #include <string.h>
  48. #include <google/protobuf/stubs/common.h>
  49. namespace google {
  50. namespace protobuf {
  51. namespace internal {
  52. // Return true if the n bytes at a equal the n bytes at b.
  53. // The regions are allowed to overlap.
  54. //
  55. // The performance is similar to the performance memcmp(), but faster for
  56. // moderately-sized inputs, or inputs that share a common prefix and differ
  57. // somewhere in their last 8 bytes. Further optimizations can be added later
  58. // if it makes sense to do so.:w
  59. inline bool memeq(const char* a, const char* b, size_t n) {
  60. size_t n_rounded_down = n & ~static_cast<size_t>(7);
  61. if (GOOGLE_PREDICT_FALSE(n_rounded_down == 0)) { // n <= 7
  62. return memcmp(a, b, n) == 0;
  63. }
  64. // n >= 8
  65. uint64 u = GOOGLE_UNALIGNED_LOAD64(a) ^ GOOGLE_UNALIGNED_LOAD64(b);
  66. uint64 v = GOOGLE_UNALIGNED_LOAD64(a + n - 8) ^ GOOGLE_UNALIGNED_LOAD64(b + n - 8);
  67. if ((u | v) != 0) { // The first or last 8 bytes differ.
  68. return false;
  69. }
  70. a += 8;
  71. b += 8;
  72. n = n_rounded_down - 8;
  73. if (n > 128) {
  74. // As of 2012, memcmp on x86-64 uses a big unrolled loop with SSE2
  75. // instructions, and while we could try to do something faster, it
  76. // doesn't seem worth pursuing.
  77. return memcmp(a, b, n) == 0;
  78. }
  79. for (; n >= 16; n -= 16) {
  80. uint64 x = GOOGLE_UNALIGNED_LOAD64(a) ^ GOOGLE_UNALIGNED_LOAD64(b);
  81. uint64 y = GOOGLE_UNALIGNED_LOAD64(a + 8) ^ GOOGLE_UNALIGNED_LOAD64(b + 8);
  82. if ((x | y) != 0) {
  83. return false;
  84. }
  85. a += 16;
  86. b += 16;
  87. }
  88. // n must be 0 or 8 now because it was a multiple of 8 at the top of the loop.
  89. return n == 0 || GOOGLE_UNALIGNED_LOAD64(a) == GOOGLE_UNALIGNED_LOAD64(b);
  90. }
  91. inline int fastmemcmp_inlined(const char *a, const char *b, size_t n) {
  92. if (n >= 64) {
  93. return memcmp(a, b, n);
  94. }
  95. const char* a_limit = a + n;
  96. while (a + sizeof(uint64) <= a_limit &&
  97. GOOGLE_UNALIGNED_LOAD64(a) == GOOGLE_UNALIGNED_LOAD64(b)) {
  98. a += sizeof(uint64);
  99. b += sizeof(uint64);
  100. }
  101. if (a + sizeof(uint32) <= a_limit &&
  102. GOOGLE_UNALIGNED_LOAD32(a) == GOOGLE_UNALIGNED_LOAD32(b)) {
  103. a += sizeof(uint32);
  104. b += sizeof(uint32);
  105. }
  106. while (a < a_limit) {
  107. int d =
  108. static_cast<int>(static_cast<uint32>(*a++) - static_cast<uint32>(*b++));
  109. if (d) return d;
  110. }
  111. return 0;
  112. }
  113. // The standard memcpy operation is slow for variable small sizes.
  114. // This implementation inlines the optimal realization for sizes 1 to 16.
  115. // To avoid code bloat don't use it in case of not performance-critical spots,
  116. // nor when you don't expect very frequent values of size <= 16.
  117. inline void memcpy_inlined(char *dst, const char *src, size_t size) {
  118. // Compiler inlines code with minimal amount of data movement when third
  119. // parameter of memcpy is a constant.
  120. switch (size) {
  121. case 1: memcpy(dst, src, 1); break;
  122. case 2: memcpy(dst, src, 2); break;
  123. case 3: memcpy(dst, src, 3); break;
  124. case 4: memcpy(dst, src, 4); break;
  125. case 5: memcpy(dst, src, 5); break;
  126. case 6: memcpy(dst, src, 6); break;
  127. case 7: memcpy(dst, src, 7); break;
  128. case 8: memcpy(dst, src, 8); break;
  129. case 9: memcpy(dst, src, 9); break;
  130. case 10: memcpy(dst, src, 10); break;
  131. case 11: memcpy(dst, src, 11); break;
  132. case 12: memcpy(dst, src, 12); break;
  133. case 13: memcpy(dst, src, 13); break;
  134. case 14: memcpy(dst, src, 14); break;
  135. case 15: memcpy(dst, src, 15); break;
  136. case 16: memcpy(dst, src, 16); break;
  137. default: memcpy(dst, src, size); break;
  138. }
  139. }
  140. } // namespace internal
  141. } // namespace protobuf
  142. } // namespace google
  143. #endif // GOOGLE_PROTOBUF_STUBS_FASTMEM_H_