diff --git a/CMakeLists.txt b/CMakeLists.txt index ed1b6f8..e45fbef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,28 +26,6 @@ if (NOT CMAKE_BUILD_TYPE) endif(NOT CMAKE_BUILD_TYPE) -# Begin Google local change - -# We use an in tree copy of boringSSL by default. -option(USE_BORINGSSL "build with boringSSL" OFF) - -option(IN_GOOGLE3 "building in google3" OFF) - -# The vendors subdirectories may not be present. -find_path( - VENDORS_AMD_PATH - NAMES amd.cc - PATHS ${CMAKE_CURRENT_SOURCE_DIR}/vendors/amd - NO_DEFAULT_PATH -) - -find_path( - VENDORS_INTEL_PATH - NAMES intel.cc - PATHS ${CMAKE_CURRENT_SOURCE_DIR}/vendors/intel - NO_DEFAULT_PATH -) -# End Google local change # Config header configure_file ( @@ -78,24 +56,20 @@ set(CMAKE_CXX_EXTENSIONS OFF) # we want c++17 not gnu++17 add_executable(cpu_check cpu_check.cc) add_executable(crc32c_test crc32c_test.cc) +# Third party library - available as git submodule add_library(farmhash third_party/farmhash/src/farmhash.cc) + +add_library(avx avx.cc) +add_library(compressor compressor.cc) add_library(crc32c crc32c.c) +add_library(crypto crypto.cc) add_library(fvt_controller fvt_controller.cc) +add_library(hasher hasher.cc) +add_library(malign_buffer malign_buffer.cc) +add_library(pattern_generator pattern_generator.cc) +add_library(silkscreen silkscreen.cc) add_library(utils utils.cc) -# Begin Google local change -if (VENDORS_AMD_PATH) - add_library(amd ${VENDORS_AMD_PATH}/amd.cc) - add_library(hsmp ${VENDORS_AMD_PATH}/hsmp.cc) - target_link_libraries(amd hsmp pci) - set(VENDORS_LIBS ${VENDORS_LIBS} amd) -endif(VENDORS_AMD_PATH) - -if (VENDORS_INTEL_PATH) - add_library(intel ${VENDORS_INTEL_PATH}/intel.cc) - set(VENDORS_LIBS ${VENDORS_LIBS} intel) -endif(VENDORS_INTEL_PATH) -# End Google local change include(CheckCXXCompilerFlag) check_cxx_compiler_flag("-march=sandybridge" ARCH_SANDYBRIDGE) @@ -104,16 +78,20 @@ if(ARCH_SANDYBRIDGE) target_compile_options(crc32c PUBLIC -march=sandybridge) endif(ARCH_SANDYBRIDGE) -target_link_libraries(cpu_check crc32c farmhash) -# Begin Google local change -target_link_libraries(cpu_check fvt_controller ${VENDORS_LIBS} utils) -# End Google local change -target_link_libraries(crc32c_test crc32c) - if (BUILD_STATIC) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") endif(BUILD_STATIC) +# Needs abseil +find_package(absl REQUIRED) +target_link_libraries(compressor absl::status absl::strings) +target_link_libraries(crypto absl::status absl::strings) +target_link_libraries(fvt_controller absl::strings) +target_link_libraries(malign_buffer absl::strings) +target_link_libraries(silkscreen absl::status absl::strings) +target_link_libraries(utils absl::strings) +target_link_libraries(cpu_check absl::failure_signal_handler absl::statusor absl::strings absl::symbolize) + # Needs pthreads find_package(Threads REQUIRED) target_link_libraries(cpu_check Threads::Threads) @@ -125,22 +103,17 @@ if(ZLIB_INCLUDE_DIRS) endif(ZLIB_INCLUDE_DIRS) if(ZLIB_LIBRARIES) target_link_libraries(cpu_check ${ZLIB_LIBRARIES}) + target_link_libraries(compressor ${ZLIB_LIBRARIES}) + target_link_libraries(hasher ${ZLIB_LIBRARIES}) endif(ZLIB_LIBRARIES) -# Begin Google local change -if(USE_BORINGSSL) - set(BORINGSSL_PATH ${CMAKE_CURRENT_SOURCE_DIR}/boringssl) - add_subdirectory(${BORINGSSL_PATH}) - set(BORINGSSL_INCLUDE_DIRS ${BORINGSSL_PATH}/include) - include_directories("${BORINGSSL_PATH}/include") - target_link_libraries(cpu_check ssl crypto) -else(USE_BORINGSSL) -# End Google local change # Needs OpenSSL find_package (OpenSSL REQUIRED) include_directories(${OPENSSL_INCLUDE_DIRS}) target_link_libraries(cpu_check ${OPENSSL_LIBRARIES}) +target_link_libraries(crypto ${OPENSSL_LIBRARIES}) +target_link_libraries(hasher ${OPENSSL_LIBRARIES}) # Static linking of OpenSSL may require -ldl, link it if found. find_library (dl dl) @@ -148,8 +121,18 @@ if(dl) target_link_libraries(cpu_check dl) endif(dl) -# Begin Google local change -endif(USE_BORINGSSL) -# End Google local change + + +# link malign_buffer first as it has a lot of dependencies. +target_link_libraries(malign_buffer utils) + +target_link_libraries(crc32c_test crc32c) +target_link_libraries(compressor malign_buffer) +target_link_libraries(crypto malign_buffer) +target_link_libraries(hasher crc32c farmhash malign_buffer utils) +target_link_libraries(pattern_generator malign_buffer) +target_link_libraries(silkscreen utils) + +target_link_libraries(cpu_check avx compressor crc32c crypto fvt_controller hasher malign_buffer pattern_generator silkscreen utils) install (TARGETS cpu_check DESTINATION bin) diff --git a/README.md b/README.md index fab88a5..f058d83 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,11 @@ Designed to run under Unix/Linux OS. * cmake: https://cmake.org/ * zlib -* OpenSSL/BoringSSL +* OpenSSL +* Abseil-cpp: https://github.com/abseil/abseil-cpp + +Note that Abseil must be built with the C++17 standard and include the +StatusOr package (release 2020_09_23 or later). ## Building diff --git a/avx.cc b/avx.cc new file mode 100644 index 0000000..d33f8e5 --- /dev/null +++ b/avx.cc @@ -0,0 +1,194 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "avx.h" + +#if defined(__i386__) || defined(__x86_64__) +#include +#endif + +#if defined(__i386__) || defined(__x86_64__) +#define X86_TARGET_ATTRIBUTE(s) __attribute__((target(s))) +#else +#define X86_TARGET_ATTRIBUTE(s) +#endif + +#if defined(__i386__) || defined(__x86_64__) + +bool Avx::can_do_avx() { + __builtin_cpu_init(); + return __builtin_cpu_supports("avx"); +} + +bool Avx::can_do_avx512f() { + __builtin_cpu_init(); + return __builtin_cpu_supports("avx512f"); +} + +bool Avx::can_do_fma() { + __builtin_cpu_init(); + return __builtin_cpu_supports("fma"); +} + +#else + +bool Avx::can_do_avx() { return false; } +bool Avx::can_do_avx512f() { return false; } +bool Avx::can_do_fma() { return false; } + +#endif + +std::string Avx::MaybeGoHot() { + if (std::uniform_int_distribution(0, 1)(rng_)) { + // Don't provoke. + level_ = 0; + return ""; + } + if (can_do_avx512f()) { + // Processor supports both AVX and AVX512. + level_ = std::uniform_int_distribution(0, 1)(rng_) ? 3 : 5; + } else { + // Processor supports only AVX. + level_ = 3; + } + return BurnIfAvxHeavy(); +} + +std::string Avx::BurnIfAvxHeavy() { + if (level_ == 3) { + return can_do_fma() ? Avx256FMA(kIterations) : Avx256(kIterations); + } + if (level_ == 5) { + return Avx512(kIterations); + } + return ""; +} + +// See notes for Avx512 below +X86_TARGET_ATTRIBUTE("avx") +std::string Avx::Avx256(int rounds) { +#if (defined(__i386__) || defined(__x86_64__)) + const __m256d minus_four = _mm256_set1_pd(-4.0); + __m256d x[4]; + for (int k = 0; k < 4; k++) { + x[k] = + _mm256_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); + } + double *gross_x[4] = { + reinterpret_cast(&x[0]), + reinterpret_cast(&x[1]), + reinterpret_cast(&x[2]), + reinterpret_cast(&x[3]), + }; + for (int i = 0; i < rounds; i++) { + __m256d a[4]; + a[0] = _mm256_sub_pd(_mm256_mul_pd(x[0], x[0]), x[0]); + a[1] = _mm256_sub_pd(_mm256_mul_pd(x[1], x[1]), x[1]); + a[2] = _mm256_sub_pd(_mm256_mul_pd(x[2], x[2]), x[2]); + a[3] = _mm256_sub_pd(_mm256_mul_pd(x[3], x[3]), x[3]); + x[0] = _mm256_mul_pd(minus_four, a[0]); + x[1] = _mm256_mul_pd(minus_four, a[1]); + x[2] = _mm256_mul_pd(minus_four, a[2]); + x[3] = _mm256_mul_pd(minus_four, a[3]); + } + for (int k = 1; k < 4; k++) { + for (int i = 0; i < 4; i++) { + if (gross_x[k][i] != gross_x[k][0]) { + return "avx256 pd"; + } + } + } +#endif + return ""; +} + +// See notes for Avx512 below +X86_TARGET_ATTRIBUTE("avx,fma") +std::string Avx::Avx256FMA(int rounds) { +#if (defined(__i386__) || defined(__x86_64__)) + const __m256d minus_four = _mm256_set1_pd(-4.0); + __m256d x[4]; + for (int k = 0; k < 4; k++) { + x[k] = + _mm256_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); + } + double *gross_x[4] = { + reinterpret_cast(&x[0]), + reinterpret_cast(&x[1]), + reinterpret_cast(&x[2]), + reinterpret_cast(&x[3]), + }; + for (int i = 0; i < rounds; i++) { + __m256d a[4]; + a[0] = _mm256_fmsub_pd(x[0], x[0], x[0]); + a[1] = _mm256_fmsub_pd(x[1], x[1], x[1]); + a[2] = _mm256_fmsub_pd(x[2], x[2], x[2]); + a[3] = _mm256_fmsub_pd(x[3], x[3], x[3]); + x[0] = _mm256_mul_pd(minus_four, a[0]); + x[1] = _mm256_mul_pd(minus_four, a[1]); + x[2] = _mm256_mul_pd(minus_four, a[2]); + x[3] = _mm256_mul_pd(minus_four, a[3]); + } + for (int k = 1; k < 4; k++) { + for (int i = 0; i < 4; i++) { + if (gross_x[k][i] != gross_x[k][0]) { + return "avx256 pd"; + } + } + } +#endif + return ""; +} + +// Interleave AVX512 parallel calculation of iterates of f(x) = 4x(1-x). +// Hope compiler too dumb to see through this. +X86_TARGET_ATTRIBUTE("avx512f") +std::string Avx::Avx512(int rounds) { +#if (defined(__i386__) || defined(__x86_64__)) + const __m512d minus_four = _mm512_set1_pd(-4.0); + __m512d x[4]; + for (int k = 0; k < 4; k++) { + x[k] = + _mm512_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); + } + + double *gross_x[4] = { + reinterpret_cast(&x[0]), + reinterpret_cast(&x[1]), + reinterpret_cast(&x[2]), + reinterpret_cast(&x[3]), + }; + + for (int i = 0; i < rounds; i++) { + __m512d a[4]; + a[0] = _mm512_fmsub_pd(x[0], x[0], x[0]); + a[1] = _mm512_fmsub_pd(x[1], x[1], x[1]); + a[2] = _mm512_fmsub_pd(x[2], x[2], x[2]); + a[3] = _mm512_fmsub_pd(x[3], x[3], x[3]); + x[0] = _mm512_mul_pd(minus_four, a[0]); + x[1] = _mm512_mul_pd(minus_four, a[1]); + x[2] = _mm512_mul_pd(minus_four, a[2]); + x[3] = _mm512_mul_pd(minus_four, a[3]); + } + + for (int k = 1; k < 4; k++) { + for (int i = 0; i < 7; i++) { + if (gross_x[k][i] != gross_x[k][0]) { + return "avx512 pd"; + } + } + } +#endif + return ""; +} diff --git a/avx.h b/avx.h new file mode 100644 index 0000000..7e2b473 --- /dev/null +++ b/avx.h @@ -0,0 +1,55 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#ifndef THIRD_PARTY_CPU_CHECK_AVX_H_ +#define THIRD_PARTY_CPU_CHECK_AVX_H_ + +// x86 AVX usage has complicated core power effects. This code tries +// to provoke some power transitions that don't otherwise happen. +// While it's at it, it lightly checks results, but that's not the central +// goal. ToDo: maybe toughen the correctness checking. +// +// The power policies are governed by a number of opaque parameters; this code +// is based on a lot of guesses. +// +// Not thread safe. +class Avx { + public: + static bool can_do_avx(); + static bool can_do_avx512f(); + static bool can_do_fma(); + + Avx() {} + + // Activate AVX depending on throw of the dice. + // Returns syndrome if computational error detected, empty string otherwise. + std::string MaybeGoHot(); + + // Does a bit of computing if in a "hot" mode. + // Returns syndrome if computational error detected, empty string otherwise. + std::string BurnIfAvxHeavy(); + + private: + constexpr static int kIterations = 5000; + std::string Avx256(int rounds); + std::string Avx256FMA(int rounds); + std::string Avx512(int rounds); + int level_ = 0; + std::knuth_b rng_; +}; + +#endif // THIRD_PARTY_CPU_CHECK_AVX_H_ diff --git a/compressor.cc b/compressor.cc new file mode 100644 index 0000000..4348f81 --- /dev/null +++ b/compressor.cc @@ -0,0 +1,56 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "compressor.h" + +#include "absl/status/status.h" +#include "absl/strings/str_format.h" +#include + +namespace cpu_check { + +absl::Status Zlib::Compress(const MalignBuffer &m, + MalignBuffer *compressed) const { + uLongf olen = compressBound(m.size()); + compressed->resize(olen); + int err = compress2(reinterpret_cast(compressed->data()), &olen, + reinterpret_cast(m.data()), m.size(), + Z_BEST_SPEED); + if (err != Z_OK) { + return absl::Status( + absl::StatusCode::kInternal, + absl::StrFormat("Zlib compression failed: %d srcLen: %d destLen: %d", + err, m.size(), olen)); + } + compressed->resize(olen); + return absl::OkStatus(); +} + +absl::Status Zlib::Decompress(const MalignBuffer &compressed, + MalignBuffer *m) const { + uLongf olen = m->size(); + int err = uncompress(reinterpret_cast(m->data()), &olen, + reinterpret_cast(compressed.data()), + compressed.size()); + if (err != Z_OK) { + return absl::Status( + absl::StatusCode::kInternal, + absl::StrFormat("Zlib decompression failed: %d srcLen: %d destLen: %d", + err, compressed.size(), olen)); + } + m->resize(olen); + return absl::OkStatus(); +} + +}; // namespace cpu_check diff --git a/compressor.h b/compressor.h new file mode 100644 index 0000000..b8c66b2 --- /dev/null +++ b/compressor.h @@ -0,0 +1,49 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_COMPRESSOR_H_ +#define THIRD_PARTY_CPU_CHECK_COMPRESSOR_H_ + +#include + +#include "malign_buffer.h" +#include "absl/status/status.h" + +namespace cpu_check { + +class Compressor { + public: + virtual ~Compressor() {} + virtual std::string Name() const = 0; + + // Compresses 'm' into 'compressed'. + virtual absl::Status Compress(const MalignBuffer &m, + MalignBuffer *compressed) const = 0; + + // Decompresses 'compressed' into 'm'. + virtual absl::Status Decompress(const MalignBuffer &compressed, + MalignBuffer *m) const = 0; +}; + +class Zlib : public Compressor { + public: + std::string Name() const override { return "ZLIB"; } + absl::Status Compress(const MalignBuffer &m, + MalignBuffer *compressed) const override; + absl::Status Decompress(const MalignBuffer &compressed, + MalignBuffer *m) const override; +}; + +}; // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_COMPRESSOR_H_ diff --git a/config.h.in b/config.h.in index a6641e8..e70cd9d 100644 --- a/config.h.in +++ b/config.h.in @@ -1,8 +1,2 @@ #define cpu_check_VERSION "@cpu_check_VERSION@" -// Begin Google local change -#cmakedefine IN_GOOGLE3 -#cmakedefine USE_BORINGSSL -#cmakedefine VENDORS_AMD_PATH -#cmakedefine VENDORS_INTEL_PATH -// End Google local change diff --git a/cpu_check.cc b/cpu_check.cc index 32c46e5..f250823 100644 --- a/cpu_check.cc +++ b/cpu_check.cc @@ -12,164 +12,63 @@ // See the License for the specific language governing permissions and // limitations under the License. -#undef NDEBUG -#include "config.h" - -#include -#include -#include - #include #include -#if defined(__x86_64__) || defined(__i386__) -#include -#endif #include #include #include #include +#include +#include #include -#include -#ifdef IN_GOOGLE3 -#include "third_party/zlib/zlib.h" -#else -#include -#endif #include #include -#include #include #include #include #include #include -#include #include -#include #include #include #include #include #include #include -#include #include #include #include #include -#ifdef IN_GOOGLE3 -#include "third_party/openssl/crypto.h" -#include "third_party/openssl/evp.h" -#else -#include -#include -#endif +#include "config.h" +#include "absl/debugging/failure_signal_handler.h" +#include "absl/debugging/symbolize.h" +#include "absl/status/status.h" +#include "absl/strings/str_format.h" -#ifdef IN_GOOGLE3 -#include "third_party/absl/debugging/failure_signal_handler.h" -#include "third_party/absl/debugging/symbolize.h" -#endif - -#include "crc32c.h" -#include "third_party/farmhash/src/farmhash.h" +#include "avx.h" +#include "compressor.h" +#include "crypto.h" #include "fvt_controller.h" +#include "hasher.h" #include "log.h" +#include "malign_buffer.h" +#include "pattern_generator.h" +#include "silkscreen.h" +#include "stopper.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "utils.h" -#if defined(__i386__) || defined(__x86_64__) -#define X86_TARGET_ATTRIBUTE(s) __attribute__ ((target (s))) -#else -#define X86_TARGET_ATTRIBUTE(s) -#endif - #undef HAS_FEATURE_MEMORY_SANITIZER #if defined(__has_feature) -# if __has_feature(memory_sanitizer) +#if __has_feature(memory_sanitizer) #define HAS_FEATURE_MEMORY_SANITIZER -# endif +#endif #endif -// Helper to make MSAN happy. NOP if memory sanitizer is not enabled. -void InitializeMemoryForSanitizer(char* addr, size_t size) { -#ifdef HAS_FEATURE_MEMORY_SANITIZER - std::default_random_engine rnd; - std::uniform_int_distribution dist(std::numeric_limits::min(), - std::numeric_limits::max()); - for (size_t i = 0; i < size; i++) { - addr[i] = dist(rnd); - } -#endif -} - -inline void __movsb(char *dst, const char *src, size_t size) { -#if defined(__i386__) || defined(__x86_64__) - __asm__ __volatile__("rep movsb" - : "+D"(dst), "+S"(src), "+c"(size) - : - : "memory"); -#else - LOG(FATAL) << "Cannot rep;movsb"; -#endif -} - -inline void __stosb(void *dst, unsigned char c, size_t size) { -#if defined(__i386__) || defined(__x86_64__) - __asm__ __volatile__("rep stosb" - : "+D"(dst), "+c"(size) - : "a"(c) - : "memory"); -#else - LOG(FATAL) << "Cannot rep;stosb"; -#endif -} - -inline void __sse_128_memcpy(char *dst, const char *src, size_t size) { -#if (defined(__i386__) || defined(__x86_64__)) - size_t blks = size / 16; - for (int i = 0; i < blks; i++) { - _mm_storeu_si128( - reinterpret_cast<__m128i *>(dst) + i, - _mm_loadu_si128(reinterpret_cast(src) + i)); - } - memcpy(dst + blks * 16, src + blks * 16, size - blks * 16); -#else - LOG(FATAL) << "SSE not available"; -#endif -} - -X86_TARGET_ATTRIBUTE("avx") -inline void __avx_256_memcpy(char *dst, const char *src, size_t size) { -#if (defined(__i386__) || defined(__x86_64__)) - size_t blks = size / 32; - for (int i = 0; i < blks; i++) { - _mm256_storeu_si256( - reinterpret_cast<__m256i *>(dst) + i, - _mm256_loadu_si256(reinterpret_cast(src) + i)); - } - memcpy(dst + blks * 32, src + blks * 32, size - blks * 32); -#else - LOG(FATAL) << "x86 only"; -#endif -} - -X86_TARGET_ATTRIBUTE("avx512f") -inline void __avx_512_memcpy(char *dst, const char *src, size_t size) { -#if (defined(__i386__) || defined(__x86_64__)) - size_t blks = size / 64; - for (int i = 0; i < blks; i++) { - _mm512_storeu_si512( - reinterpret_cast<__m512i *>(dst) + i, - _mm512_loadu_si512(reinterpret_cast(src) + i)); - } - memcpy(dst + blks * 64, src + blks * 64, size - blks * 64); -#else - LOG(FATAL) << "x86 only"; -#endif -} - -static const double t0 = TimeInSeconds(); +using cpu_check::MalignBuffer; static void SleepForMillis(int64_t millis) { // LOG(INFO) << "sleeping " << millis; @@ -184,25 +83,6 @@ static void SleepForMillis(int64_t millis) { } } -static long pagesize = sysconf(_SC_PAGESIZE); -static long cache_line_size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); - -std::string HexData(const char *s, uint32_t l) { - const char d[16] = {'0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - std::string o; - o.resize(l << 1); - for (uint32_t i = 0; i < l; i++) { - uint8_t b = s[i]; - o[(i << 1)] = d[(b >> 4) & 0xf]; - o[(i << 1) + 1] = d[b & 0xf]; - } - return o; -} - -std::string HexStr(const std::string &s) { return HexData(s.data(), s.size()); } - -std::atomic_bool exiting(false); std::atomic_uintmax_t errorCount(0); std::atomic_uintmax_t successCount(0); static constexpr uintmax_t kErrorLimit = 2000; @@ -213,45 +93,24 @@ const bool is_x86 = true; const bool is_x86 = false; #endif -// So-called Logistic Map with parameter 4.0. -// Floating point approximation aside, if 0 < v < 1 then 0 < ChaoticF1(v) < 1. -static inline double ChaoticF1(double v) { - return 4.0 * v * (1.0 - v); -} - #if defined(__i386__) || defined(__x86_64__) -static bool can_do_avx() { - __builtin_cpu_init(); - return __builtin_cpu_supports("avx"); -} - -static bool can_do_avx512f() { - __builtin_cpu_init(); - return __builtin_cpu_supports("avx512f"); -} - -static bool can_do_fma() { - __builtin_cpu_init(); - return __builtin_cpu_supports("fma"); -} static bool can_do_fvt() { - return geteuid() == 0; // need write access to MSRs. + return geteuid() == 0; // need write access to MSRs. } #else -static bool can_do_avx() { return false; } -static bool can_do_avx512f() { return false; } -static bool can_do_fma() { return false; } + static bool can_do_fvt() { return false; } // x86-only for now. + #endif bool do_madvise = true; bool do_repmovsb = is_x86; bool do_sse_128_memcpy = is_x86; -bool do_avx_256_memcpy = can_do_avx(); -bool do_avx_512_memcpy = can_do_avx512f(); -bool do_avx_heavy = can_do_avx(); +bool do_avx_256_memcpy = Avx::can_do_avx(); +bool do_avx_512_memcpy = Avx::can_do_avx512f(); +bool do_avx_heavy = Avx::can_do_avx(); bool do_compress = true; bool do_encrypt = true; bool do_hashes = true; @@ -302,476 +161,6 @@ bool SetAffinity(int id) { return err == 0; } -std::vector ReadDict() { - // Dictionary search paths - static const char *dicts[] = { - "/usr/share/dict/words", - "words", - }; - std::vector words; - std::ifstream f; - - for (const auto &d : dicts) { - f.open(d, std::ifstream::in); - if (f.is_open()) break; - f.clear(); - } - - if (!f.is_open()) return words; - - LOG(DEBUG) << "Reading words."; - - std::string word; - while (!f.eof()) { - std::getline(f, word); - words.push_back(word); - } - f.close(); - LOG(DEBUG) << "Read " << words.size() << " words."; - std::sort(words.begin(), words.end(), - [](const std::string &a, const std::string &b) { - return a.size() < b.size(); - }); - return words; -} - -class MalignBuffer { - public: - struct PunchedHole { - std::string ToString() const; - size_t start = 0; - size_t length = 0; - unsigned char v = 0x53; - }; - - enum CopyMethod { - kMemcpy, - kRepMov, - kSseBy128, - kAvxBy256, - kAvxBy512, - }; - - static std::string ToString(CopyMethod m) { - switch (m) { - case kMemcpy: - return "memcpy"; - case kRepMov: - return "rep;mov"; - case kSseBy128: - return "sse:128"; - case kAvxBy256: - return "avx:256"; - case kAvxBy512: - return "avx:512"; - } - } - - // Provides buffer with specified alignment. - MalignBuffer(size_t capacity) - : capacity_(capacity), - base_address_(aligned_alloc(pagesize, capacity + pagesize)) { - assert(base_address_); - // There are lots of places that use unitialized MalignBuffer. So just - // fill some pseudo-random bytes if cpu_check is compiled with msan. - InitializeMemoryForSanitizer(static_cast(base_address_), capacity_); - } - ~MalignBuffer() { free(base_address_); } - - // REQUIRES: alignment_offset + length <= capacity_. - void Initialize(size_t alignment_offset, size_t length); - - const char* data() const { return buffer_address_; } - char* data() { return buffer_address_; } - size_t size() const { return length_; } - - // REQUIRES alignment_offset + length <= capacity_. - void resize(size_t length); - - // Compares 'this' to 'that' returning empty string if identical. - // If not identical, returns a syndrome, currently Hamming distance, - // corrupted subrange bounds, and the diffs. - std::string Syndrome(const MalignBuffer& that) const; - - // Validated data copy from source to 'this'. - // 'this' must be appropriately sized. - // Returns syndrome upon copy failure. - std::string CopyFrom(const MalignBuffer& that, CopyMethod m); - - // Unvalidated copy to 'this'. - void CopyFrom(const char* src, size_t length, CopyMethod m); - void CopyFrom(size_t pos, const char* src, size_t length, CopyMethod m); - - // Randomly flushes cache lines. - void RandomFlush(std::knuth_b* rng) const; - - // Conventional or rep;sto memset operation, according to 'use_rep_stos'. - void Memset(size_t offset, unsigned char v, size_t length, bool use_rep_stos); - - // Memsets buffer to 'hole.v', using rep;stos operation if - // 'use_rep_stos' set; - void PunchHole(const PunchedHole& hole, bool use_rep_stos); - - private: - std::string CorruptionSyndrome(const MalignBuffer& that) const; - std::string CrackId(uint64_t) const; - - const size_t capacity_; - void* base_address_ = nullptr; - - size_t alignment_offset_ = 0; - size_t length_ = 0; // Usable length - char* buffer_address_ = nullptr; -}; - -void MalignBuffer::Initialize(size_t alignment_offset, size_t length) { - assert(alignment_offset + length <= capacity_); - alignment_offset_ = alignment_offset; - length_ = length; - buffer_address_ = static_cast(base_address_) + alignment_offset_; -} - -void MalignBuffer::resize(size_t length) { - Initialize(alignment_offset_, length); -} - -std::string MalignBuffer::CopyFrom(const MalignBuffer& that, CopyMethod m) { - CopyFrom(that.data(), that.size(), m); - return Syndrome(that); -} - -void MalignBuffer::CopyFrom(const char* src, size_t length, CopyMethod m) { - assert(size() == length); - CopyFrom(0, src, length, m); -} - -void MalignBuffer::CopyFrom(size_t pos, const char* src, size_t length, - CopyMethod m) { -assert(pos + length <= size()); -switch (m) { - case kMemcpy: - // Assumes memcpy doesn't use rep;movsb; false in lots of environments. - memcpy(data() + pos, src, length); - break; - case kRepMov: - __movsb(data() + pos, src, length); - break; - case kSseBy128: - __sse_128_memcpy(data() + pos, src, length); - break; - case kAvxBy256: - __avx_256_memcpy(data() + pos, src, length); - break; - case kAvxBy512: - __avx_512_memcpy(data() + pos, src, length); - break; -} -} - -std::string MalignBuffer::Syndrome(const MalignBuffer& that) const { - std::stringstream s; - std::string syndrome = CorruptionSyndrome(that); - if (syndrome.empty()) return ""; - s << syndrome << ", \"this\": \"" << static_cast(data()) - << "\", " - << "\"that\": \"" << static_cast(that.data()) << "\""; - return s.str(); -} - -std::string MalignBuffer::CorruptionSyndrome(const MalignBuffer& that) const { - std::stringstream s; - if (size() != that.size()) { - s << Json("unequalSizeThis", size()) << ", " - << Json("unequalSizeThat", that.size()); - return s.str(); - } - bool failed_memcmp = memcmp(data(), that.data(), that.size()); - - int wrong_bytes = 0; - int wrong_bits = 0; - int byte_faults = 0; - int first_wrong = INT_MAX; - int last_wrong = INT_MIN; - std::vector lane_errors(8, 0); - for (size_t i = 0; i < size(); i++) { - unsigned char a = *(data() + i); - unsigned char b = *(that.data() + i); - unsigned char d = a ^ b; - if (d) { - first_wrong = std::min(first_wrong, i); - last_wrong = std::max(last_wrong, i); - byte_faults |= d; - wrong_bytes++; - wrong_bits += __builtin_popcount(d); - for (size_t i = 0; i < 8; i++) { - if ((d >> i) & 1) { - lane_errors[i]++; - } - } - } - } - if (wrong_bits || wrong_bytes) { - const int range_width = (last_wrong - first_wrong) + 1; - s << Json("cmpResult", - (failed_memcmp ? "Failed_Memcmp" : "**Passed_Memcmp**")) - << ", " << Json("wrongByteCount", wrong_bytes) << ", " - << Json("wrongBitCount", wrong_bits) << ", " - << Json("corruptionWidth", range_width) << ", " - << Json("corruptStart", first_wrong) << ", " - << Json("corruptByteBitMask", byte_faults) << ", " - << "\"byBitLane\": ["; - for (size_t i = 0; i < 8; i++) { - if (i) s << ", "; - s << lane_errors[i]; - } - s << " ] "; - // Dump up to 64 corrupted locations. - std::stringstream dump; - dump << " \"byteErrors\": [ " << std::hex; - uint64_t buf_a = 0; - uint64_t buf_b = 0; - for (size_t k = 0; k < std::min(64, range_width); k++) { - uint8_t a = *(data() + first_wrong + k); - uint8_t b = *(that.data() + first_wrong + k); - if (k) dump << ", "; - dump << "[ " << std::setw(2) << "\"0x" << static_cast(a) << "\", " - << std::setw(2) << "\"0x" << static_cast(b) << "\" "; - buf_a = (buf_a >> 8) | static_cast(a) << 56; - buf_b = (buf_b >> 8) | static_cast(b) << 56; - if ((k >= 7) && (7 == ((first_wrong + k) % 8))) { - dump << ", " << CrackId(buf_a) << ", " << CrackId(buf_b); - buf_a = 0; - buf_b = 0; - } - dump << " ]"; - } - dump << " ] "; - return s.str() + ", " + dump.str(); - } else { - if (!failed_memcmp) return ""; - return Json("cmpResult", "**Failed_Memcmp**"); - } -} - -std::string MalignBuffer::CrackId(uint64_t v) const { - std::stringstream s; - s << std::hex << " [\"0x" << std::setw(4) << (v >> 48) << "\", \"0x" - << std::setw(6) << ((v >> 24) & 0xffffff) << "\", \"0x" << std::setw(6) - << (v & 0xffffff) << "\"]"; - return s.str(); -} - -void MalignBuffer::RandomFlush(std::knuth_b* rng) const { -#if defined(__i386__) || defined(__x86_64__) - // Note: no barriers used. - const char* p = buffer_address_ + alignment_offset_; - while (p < buffer_address_ + length_) { - if (std::uniform_int_distribution(0, 1)(*rng)) { - __builtin_ia32_clflush(p); - } - p += cache_line_size; - } -#endif -} - -std::string MalignBuffer::PunchedHole::ToString() const { - if (length) { - return JsonRecord("hole", - Json("start", start) + ", " + Json("length", length) - + ", " + Json("v", static_cast(v))); - } else { - return JsonNull("hole"); - } -} - -void MalignBuffer::Memset(size_t offset, unsigned char v, size_t length, - bool use_rep_stos) { - if (use_rep_stos) { - __stosb(data() + offset, v, length); - } else { - memset(data() + offset, v, length); - } -} - -void MalignBuffer::PunchHole(const PunchedHole& hole, bool use_rep_stos) { - if (hole.length) { - Memset(hole.start, hole.v, hole.length, use_rep_stos); - } -} - -// x86 AVX usage has complicated core power effects. This code tries -// to provoke some power transitions that don't otherwise happen. -// While it's at it, it lightly checks results, but that's not the central -// goal. ToDo: maybe toughen the correctness checking. -// -// The power policies are governed by a number of opaque parameters; this code -// is based on a lot of guesses. - -class Avx { - public: - Avx() {} - - // Activate AVX depending on throw of the dice. - // Returns syndrome if computational error detected. - std::string MaybeGoHot(); - - // Does a bit of computing if in a "hot" mode. - // Returns syndrome if computational error detected. - std::string BurnIfAvxHeavy(); - - private: - constexpr static int kIterations = 5000; - std::string Avx256(int rounds); - std::string Avx256FMA(int rounds); - std::string Avx512(int rounds); - int level_ = 0; - std::knuth_b rng_; -}; - -std::string Avx::MaybeGoHot() { - if (std::uniform_int_distribution(0, 1)(rng_)) { - // Don't provoke. - level_ = 0; - return ""; - } - if (can_do_avx512f()) { - // Processor supports both AVX and AVX512. - level_ = std::uniform_int_distribution(0, 1)(rng_) ? 3 : 5; - } else { - // Processor supports only AVX. - level_ = 3; - } - return BurnIfAvxHeavy(); -} - -std::string Avx::BurnIfAvxHeavy() { - if (level_ == 3) { - return can_do_fma() ? Avx256FMA(kIterations) : Avx256(kIterations); - } - if (level_ == 5) { - return Avx512(kIterations); - } - return ""; -} - -// See notes for Avx512 below -X86_TARGET_ATTRIBUTE("avx") -std::string Avx::Avx256(int rounds) { -#if (defined(__i386__) || defined(__x86_64__)) - const __m256d minus_four = _mm256_set1_pd(-4.0); - __m256d x[4]; - for (int k = 0; k < 4; k++) { - x[k] = - _mm256_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); - } - double *gross_x[4] = { - reinterpret_cast(&x[0]), - reinterpret_cast(&x[1]), - reinterpret_cast(&x[2]), - reinterpret_cast(&x[3]), - }; - for (int i = 0; i < rounds; i++) { - __m256d a[4]; - a[0] = _mm256_sub_pd(_mm256_mul_pd(x[0], x[0]), x[0]); - a[1] = _mm256_sub_pd(_mm256_mul_pd(x[1], x[1]), x[1]); - a[2] = _mm256_sub_pd(_mm256_mul_pd(x[2], x[2]), x[2]); - a[3] = _mm256_sub_pd(_mm256_mul_pd(x[3], x[3]), x[3]); - x[0] = _mm256_mul_pd(minus_four, a[0]); - x[1] = _mm256_mul_pd(minus_four, a[1]); - x[2] = _mm256_mul_pd(minus_four, a[2]); - x[3] = _mm256_mul_pd(minus_four, a[3]); - } - for (int k = 1; k < 4; k++) { - for (int i = 0; i < 4; i++) { - if (gross_x[k][i] != gross_x[k][0]) { - return "avx256 pd"; - } - } - } -#endif - return ""; -} - -// See notes for Avx512 below -X86_TARGET_ATTRIBUTE("avx,fma") -std::string Avx::Avx256FMA(int rounds) { -#if (defined(__i386__) || defined(__x86_64__)) - const __m256d minus_four = _mm256_set1_pd(-4.0); - __m256d x[4]; - for (int k = 0; k < 4; k++) { - x[k] = - _mm256_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); - } - double *gross_x[4] = { - reinterpret_cast(&x[0]), - reinterpret_cast(&x[1]), - reinterpret_cast(&x[2]), - reinterpret_cast(&x[3]), - }; - for (int i = 0; i < rounds; i++) { - __m256d a[4]; - a[0] = _mm256_fmsub_pd(x[0], x[0], x[0]); - a[1] = _mm256_fmsub_pd(x[1], x[1], x[1]); - a[2] = _mm256_fmsub_pd(x[2], x[2], x[2]); - a[3] = _mm256_fmsub_pd(x[3], x[3], x[3]); - x[0] = _mm256_mul_pd(minus_four, a[0]); - x[1] = _mm256_mul_pd(minus_four, a[1]); - x[2] = _mm256_mul_pd(minus_four, a[2]); - x[3] = _mm256_mul_pd(minus_four, a[3]); - } - for (int k = 1; k < 4; k++) { - for (int i = 0; i < 4; i++) { - if (gross_x[k][i] != gross_x[k][0]) { - return "avx256 pd"; - } - } - } -#endif - return ""; -} - -// Interleave AVX512 parallel calculation of iterates of f(x) = 4x(1-x). -// Hope compiler too dumb to see through this. -X86_TARGET_ATTRIBUTE("avx512f") -std::string Avx::Avx512(int rounds) { -#if (defined(__i386__) || defined(__x86_64__)) - const __m512d minus_four = _mm512_set1_pd(-4.0); - __m512d x[4]; - for (int k = 0; k < 4; k++) { - x[k] = - _mm512_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); - } - - double *gross_x[4] = { - reinterpret_cast(&x[0]), - reinterpret_cast(&x[1]), - reinterpret_cast(&x[2]), - reinterpret_cast(&x[3]), - }; - - for (int i = 0; i < rounds; i++) { - __m512d a[4]; - a[0] = _mm512_fmsub_pd(x[0], x[0], x[0]); - a[1] = _mm512_fmsub_pd(x[1], x[1], x[1]); - a[2] = _mm512_fmsub_pd(x[2], x[2], x[2]); - a[3] = _mm512_fmsub_pd(x[3], x[3], x[3]); - x[0] = _mm512_mul_pd(minus_four, a[0]); - x[1] = _mm512_mul_pd(minus_four, a[1]); - x[2] = _mm512_mul_pd(minus_four, a[2]); - x[3] = _mm512_mul_pd(minus_four, a[3]); - } - - for (int k = 1; k < 4; k++) { - for (int i = 0; i < 7; i++) { - if (gross_x[k][i] != gross_x[k][0]) { - return "avx512 pd"; - } - } - } -#endif - return ""; -} - // Produces noise of all kinds by running intermittently. // There's a coarse cycle with four fine phases: // Phase 0: Off @@ -822,122 +211,41 @@ class NoiseScheduler { } }; -// Rudimentary coherence/uncore tester. -// Randomly assigns each slot of a seemingly shared buffer to a single tid, -// creating only "false sharing". -// Thus each slot, regardless of alignment, must obey program order unless the -// machine is broken. -// To be toughened, e.g.: -// Widen the slots a bit -// Control the sharing more tightly, e.g. each cache line split between 2 tids -// Maybe checksum the indices to distinguish core-local compute errors from -// coherence errors, but that's perhaps easier said than done effectively. -// As it stands, it may be particularly hard to localize failures. Though that's -// always going to be a bit hard, which is the point. One technique might be -// to leave this alone and to run on subsets of cores and sockets. -class Silkscreen { - public: - static constexpr size_t kSize = 1000 * 1000; // Size of buffer - - Silkscreen(const std::vector &tid_list); - ~Silkscreen() { free(buffer_address_); } - - // Writes value derived from 'round' into all slots owned by 'tid'. - // Returns number of slots written. - uint64_t WriteMySlots(int tid, uint64_t round); - - // Returns JSON-formatted error string if slot 'k' belongs to 'tid' and has - // value not properly corresponding with 'round'. - // Returns "=" if slot 'k' belongs to 'tid' and has expected value. - // Otherwise, if slot 'k' does not belong to 'tid', returns empty string. - std::string CheckMySlot(int tid, uint64_t round, size_t k) const; - - private: - int owner(size_t k) const { return owner_[k]; } - size_t size() const { return owner_.size(); } - const char* data(size_t k) const { return buffer_address_ + k; } - char* data(size_t k) { return buffer_address_ + k; } - - std::vector owner_; // const after initialization - char* buffer_address_ = nullptr; -}; - -Silkscreen::Silkscreen(const std::vector &tid_list) - : buffer_address_(static_cast(aligned_alloc( - pagesize, - pagesize * ((kSize + pagesize - 1) / pagesize) ))) { - std::knuth_b rng; - std::uniform_int_distribution dist(0, tid_list.size() - 1); - for (size_t k = 0; k < kSize; k++) { - size_t w = dist(rng); - owner_.push_back(tid_list[w]); - } -} - -uint64_t Silkscreen::WriteMySlots(int tid, uint64_t round) { - uint64_t j = 0; - for (size_t k = 0; k < kSize; k++) { - if (owner(k) == tid) { - *data(k) = static_cast(round); - j++; - } - } - return j; -} - -std::string Silkscreen::CheckMySlot(int tid, uint64_t round, size_t k) const { - if (owner(k) != tid) return ""; - const int v = *data(k); - const int w = static_cast(round); - if (v == w) return "="; - return - Json("position", k) + ", " + Json("is", v) + ", " + Json("expected", w); -} - class Worker { public: - // Does not take ownership of 'silkscreen'. - Worker(int pid, const std::vector *words, - std::vector tid_list, int tid, Silkscreen *silkscreen) - : pid_(pid), tid_(tid), words_(words), tid_list_(tid_list), - silkscreen_(silkscreen), rndeng_(std::random_device()()) { - } + // Does not take ownership of 'silkscreen' or 'stopper'. + Worker(int pid, std::vector tid_list, int tid, + cpu_check::Silkscreen *silkscreen, Stopper *stopper) + : pid_(pid), + tid_(tid), + tid_list_(tid_list), + silkscreen_(silkscreen), + stopper_(stopper), + rndeng_(std::random_device()()) {} ~Worker() {} void Run(); private: static constexpr size_t kBufMin = 12; #ifdef HAVE_FEATURE_MEMORY_SANITIZER - // Use smaller buffers if cpu_check is built with msan. Otherwise - // we will time out in testing. - static constexpr size_t kBufMax = 1 << 16; // 64 KiB + // Use smaller buffers if cpu_check is built with msan. Otherwise + // we will time out in testing. + static constexpr size_t kBufMax = 1 << 16; // 64 KiB #else static constexpr size_t kBufMax = 1 << 20; // 1 MiB #endif - struct FloatingPointResults { - bool operator!=(const FloatingPointResults& other) const { - return d != other.d; - } - - double d = 0.0; - }; - - typedef struct { - const char *name; - FloatingPointResults (Worker::*func)( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer*) const; - } generatorItem; - struct BufferSet { void Alloc(std::unique_ptr *p) { - const size_t kBufCap = kBufMax + 23 * pagesize; if (!*p) { - p->reset(new MalignBuffer(kBufCap)); + // Allocate buffer larger than kBufMax because compression can, in some + // cases of random plain text, cause some expansion. + p->reset(new MalignBuffer(2 * kBufMax + 1024)); } } + // The buffers holding successive data transformations. Some transformations + // are optional, so some buffers may be unused. std::unique_ptr original; std::unique_ptr compressed; std::unique_ptr encrypted; @@ -945,73 +253,103 @@ class Worker { std::unique_ptr decrypted; std::unique_ptr decompressed; std::unique_ptr re_made; + + // The plain-text buffer that was encrypted, if encryption was performed. + MalignBuffer *pre_encrypted = nullptr; + // The result of the series of transformations up to, but not including, + // the final copy. + MalignBuffer *pre_copied = nullptr; }; - FloatingPointResults FillBufferSystematic( - uint32_t unused_seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; - FloatingPointResults FillBufferRandomData( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; - FloatingPointResults FillBufferRandomText( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; - FloatingPointResults FillBufferGrilledCheese( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; + // Computation parameters. + struct Choices { + bool madvise; + bool use_repstos; + bool exercise_floating_point; + MalignBuffer::CopyMethod copy_method; + cpu_check::PatternGenerator const *pattern_generator = nullptr; + cpu_check::Hasher const *hasher = nullptr; + size_t buf_size; + MalignBuffer::PunchedHole hole; + std::string summary; + }; - void MadviseDontNeed(const MalignBuffer &s) const; + // Various checksums produced in the course of the series of data + // transformations. + struct Checksums { + std::string hash_value; + cpu_check::FloatingPointResults floating_point_results; + cpu_check::Crypto::CryptoPurse crypto_purse; + }; + + uint64_t Seed() { return std::uniform_int_distribution()(rndeng_); } size_t Alignment(); void MaybeFlush(const MalignBuffer &s); + // Attempts to schedules CPU frequency using the Worker's. // FVTController object. Returns the scheduled frequency or // 0 if there is no FVTController available. int ScheduledMHz() const; MalignBuffer::CopyMethod CopyMethod(); std::string FVT() const; - MalignBuffer::PunchedHole PunchedHole(size_t bufsize); - FloatingPointResults GenerateData( - const generatorItem& generator, - const MalignBuffer::PunchedHole& hole, - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; + + // Returns 'Choices' that seed the data transformations. + Choices MakeChoices(BufferSet *b); + + // Performs a series of data transformations. + // Returns an error status if computation is detected to be corrupt. + absl::StatusOr DoComputations(const std::string &writer_ident, + const Choices &choices, + BufferSet *b); + + // Inverts DoComputations, checking correctness of results. + // Returns an error status if corruption is detected. + absl::Status CheckComputations(const std::string &writer_reader_ident, + const Choices &choices, + const Checksums &checksums, BufferSet *b); // Emits a failure record. // TODO: bump error count here, and maybe log, instead of at every // call site. - std::string Jfail(const std::string &err, const std::string &v) { + std::string Jfail(absl::string_view err, const absl::string_view v) { if (errorCount > error_limit) { - exiting = true; + stopper_->Stop(); LOG(INFO) << "I am quitting after " << errorCount << " errors"; } - return "{ " + JsonRecord("fail", Json("err", err) + ", " + v) + ", " + - JTag() + " }"; + return "{ " + JsonRecord("fail", absl::StrCat(Json("err", err), ", ", v)) + + ", " + JTag() + " }"; } - // Array of random data generators. - static const std::vector kGenerators; + absl::Status ReturnError(absl::string_view err, const absl::string_view v) { + return absl::Status(absl::StatusCode::kInternal, Jfail(err, v)); + } + + std::string Suspect(int tid) { + return absl::StrFormat("Suspect LPU: %d", tid); + } + + // Returns two tids to be used for checking computation. If 'do_hop', + // CheckerTids avoids duplication and avoids 'tid_' if it can. + std::vector CheckerTids(); const uint64_t pid_; const int tid_; - const std::vector *words_; const std::vector tid_list_; - Silkscreen* const silkscreen_; + cpu_check::Silkscreen *const silkscreen_; + Stopper *const stopper_; // We don't really need "good" random numbers. // std::mt19937_64 rndeng_; std::knuth_b rndeng_; uint64_t round_ = 0; + Avx avx_; + cpu_check::PatternGenerators pattern_generators_; + cpu_check::Hashers hashers_; + cpu_check::Zlib zlib_; std::unique_ptr fvt_controller_; }; -const std::vector Worker::kGenerators = { - {"SYSTEMATIC", &Worker::FillBufferSystematic}, - {"DATA", &Worker::FillBufferRandomData}, - {"TEXT", &Worker::FillBufferRandomText}, - {"CHEESE", &Worker::FillBufferGrilledCheese}, -}; - std::string Worker::FVT() const { if (fvt_controller_ == nullptr) return ""; return fvt_controller_->FVT(); @@ -1026,8 +364,8 @@ int Worker::ScheduledMHz() const { // User-specified fixed frequency. return fixed_min_frequency; } - if (!do_freq_sweep && !do_freq_hi_lo - && !fixed_min_frequency && !fixed_max_frequency) { + if (!do_freq_sweep && !do_freq_hi_lo && !fixed_min_frequency && + !fixed_max_frequency) { // Run at maximum frequency. return fvt_controller_->limit_mHz(); } @@ -1053,128 +391,7 @@ int Worker::ScheduledMHz() const { } } -// Fills 'b' with a systematic pattern. -// Returns iterate of chaotic floating point function of 'seed', with some -// reciprocal torture. -Worker::FloatingPointResults Worker::FillBufferSystematic( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const { - // Format: 2 bytes of PID, 3 bytes of round number, 3 bytes of offset. - // Note: Perhaps should be AC-modulated. Perhaps should be absolute aligned - // for easier recognition. - // Note: appropriate for LE machines only. - FloatingPointResults fp; - fp.d = std::max(seed, 2); - for (size_t i = 0; i * 8 < b->size(); i++) { - const size_t p = 8 * i; - const size_t k = std::min(8, b->size() - p); - const uint64_t v = - ((pid_ & 0xffff) << 48) - | ((round_ & 0xffffff) << 24) | (i & 0xffffff); - for (size_t m = 0; m < k; m++) { - (b->data())[p + m] = *(reinterpret_cast(&v) + m); - } - fp.d = 1.0 / ChaoticF1(1.0 / fp.d); - } - fp.d = 1.0 / fp.d; - return fp; -} - -Worker::FloatingPointResults Worker::FillBufferRandomData( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const { - std::knuth_b rng(seed); - std::uniform_int_distribution - dist(0, std::numeric_limits::max()); - size_t p = 0; - const size_t length = b->size(); - // Repeatedly append random number (one to eight) random bytes. - while (p < length) { - const size_t max_span = std::min(length - p, 8); - const size_t z = std::uniform_int_distribution(1, max_span)(rng); - const uint64_t v = dist(rng); - b->CopyFrom(p, reinterpret_cast(&v), z, copy_method); - p += z; - } - return FloatingPointResults(); -} - -Worker::FloatingPointResults Worker::FillBufferRandomText( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const { - std::knuth_b rng(seed); - std::exponential_distribution dist(20); - const size_t bufsize = b->size(); - size_t pos = 0; - while (pos < bufsize) { - const size_t r = std::min(static_cast(dist(rng) * words_->size()), - words_->size() - 1); - const auto &word = (*words_)[r]; - const size_t wordlen = word.size(); - if (pos + wordlen >= bufsize) { - break; - } - b->CopyFrom(pos, word.c_str(), wordlen, copy_method); - pos += wordlen; - if (pos < bufsize) { - b->Memset(pos, ' ', 1, use_repstos); - pos++; - } - } - // Pad with spaces - b->Memset(pos, ' ', bufsize - pos, use_repstos); - return FloatingPointResults(); -} - -// memset (conventional or rep;stos) randomly aligned, random width, randomly -// overlapped stretches of buffer. Constants aim to hit multiple times in cache -// lines and buffers. Untuned and based on nothing but hunches. -Worker::FloatingPointResults Worker::FillBufferGrilledCheese( - uint32_t seed, MalignBuffer::CopyMethod copy_method, bool use_repstos, - MalignBuffer* b) const { - std::knuth_b rng(seed); - const size_t kAdvance = 15; - const size_t kWindow = 64; - unsigned char flavor = 0; - b->Memset(0, 0, b->size(), use_repstos); - for (int base = kWindow; base < b->size(); base += kAdvance) { - if (std::uniform_int_distribution(0, 1)(rng)) continue; - flavor++; - const size_t start = - std::uniform_int_distribution(base - kWindow, base)(rng); - const size_t end = std::uniform_int_distribution(start, base)(rng); - b->Memset(start, flavor, 1 + end - start, use_repstos); - } - return FloatingPointResults(); -} - -Worker::FloatingPointResults Worker::GenerateData( - const generatorItem& generator, - const MalignBuffer::PunchedHole& hole, - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, - MalignBuffer* b) const { - const FloatingPointResults f = - (this->*generator.func)(seed, copy_method, use_repstos, b); - b->PunchHole(hole, use_repstos); - return f; -} - -// Hints to the OS to release the buffer's memory. -void Worker::MadviseDontNeed(const MalignBuffer &s) const { - // Round up the buffer start address to a page boundary. - intptr_t start = ((intptr_t) s.data() + pagesize - 1) & ~(pagesize - 1); - // Round down the buffer end address to a page boundary. - intptr_t end = ((intptr_t) (s.data() + s.size() - 1)) & ~(pagesize - 1); - if (end - start >= pagesize) { - if (madvise((char *)start, end - start, MADV_DONTNEED) == -1) { - LOG(WARN) << "tid " << tid_ - << " madvise(MADV_DONTNEED) failed: " << strerror(errno); - } - } -} - -void Worker::MaybeFlush(const MalignBuffer& s) { +void Worker::MaybeFlush(const MalignBuffer &s) { // Half the time, tell the OS to release the destination buffer. if (do_flush && std::uniform_int_distribution(0, 1)(rndeng_)) { s.RandomFlush(&rndeng_); @@ -1182,8 +399,7 @@ void Worker::MaybeFlush(const MalignBuffer& s) { } size_t Worker::Alignment() { - return do_misalign ? - std::uniform_int_distribution(0, pagesize)(rndeng_) : 0; + return do_misalign ? MalignBuffer::RandomAlignment(Seed()) : 0; } MalignBuffer::CopyMethod Worker::CopyMethod() { @@ -1202,145 +418,252 @@ MalignBuffer::CopyMethod Worker::CopyMethod() { return v[k]; } -MalignBuffer::PunchedHole Worker::PunchedHole(size_t bufsize) { - MalignBuffer::PunchedHole hole; - hole.length = - std::uniform_int_distribution( - 1, std::min(bufsize, 8192))(rndeng_); - hole.start = - std::uniform_int_distribution( - 0, bufsize - hole.length)(rndeng_); - return hole; +Worker::Choices Worker::MakeChoices(BufferSet *b) { + Choices c; + c.madvise = do_madvise && std::uniform_int_distribution(0, 1)(rndeng_); + c.copy_method = CopyMethod(); + + c.use_repstos = + do_repstosb && std::uniform_int_distribution(0, 1)(rndeng_); + + // Exercise floating point (in pattern generators) relatively rarely because + // it's expensive and it doesn't catch a lot of machines. + c.exercise_floating_point = + std::uniform_int_distribution(0, 20)(rndeng_) == 0; + + c.pattern_generator = &pattern_generators_.RandomGenerator(round_); + c.hasher = &hashers_.RandomHasher(round_); + + c.buf_size = std::uniform_int_distribution(kBufMin, kBufMax)(rndeng_); + if (!b->original) b->Alloc(&b->original); + b->original->Initialize(Alignment(), c.buf_size); + c.hole = b->original->RandomPunchedHole(Seed()); + + c.summary = absl::StrCat( + Json("pattern", c.pattern_generator->Name()), ", ", + Json("hash", c.hasher->Name()), ", ", + Json("copy", MalignBuffer::ToString(c.copy_method)), ", ", + Json("memset", c.use_repstos ? "rep;sto" : "memset"), ", ", + JsonBool("madvise", c.madvise), ", ", Json("size", c.buf_size), ", ", + Json("pid", pid_), ", ", Json("round", round_), ", ", c.hole.ToString()); + + return c; } -std::string OpenSSL_Hash(const MalignBuffer &s, const EVP_MD *type) { - EVP_MD_CTX *ctx; - ctx = EVP_MD_CTX_create(); - EVP_DigestInit_ex(ctx, type, nullptr); - std::string hash; - hash.resize(EVP_MD_CTX_size(ctx)); - InitializeMemoryForSanitizer(hash.data(), EVP_MD_CTX_size(ctx)); - EVP_DigestUpdate(ctx, s.data(), s.size()); - EVP_DigestFinal_ex(ctx, (uint8_t *)&hash[0], nullptr); - EVP_MD_CTX_destroy(ctx); - return HexStr(hash); +absl::StatusOr Worker::DoComputations( + const std::string &writer_ident, const Choices &choices, BufferSet *b) { + Checksums checksums; + if (do_avx_heavy) { + const std::string e = avx_.MaybeGoHot(); + if (!e.empty()) { + return ReturnError(e, writer_ident); + } + } + + if (do_ssl_self_check) { + auto s = cpu_check::Crypto::SelfTest(); + if (!s.ok()) { + return ReturnError(s.message(), writer_ident); + } + } + + auto s = silkscreen_->WriteMySlots(tid_, round_); + if (!s.ok()) { + return ReturnError("Silkscreen", + absl::StrCat(s.message(), ", ", writer_ident)); + } + + if (do_avx_heavy) { + // If we tried to do AVX heavy stuff. Try to run AVX heavy again to try + // to spike current. + const std::string e = avx_.BurnIfAvxHeavy(); + if (!e.empty()) { + return ReturnError(e, writer_ident); + } + } + + checksums.floating_point_results = pattern_generators_.Generate( + *choices.pattern_generator, choices.hole, round_, choices.copy_method, + choices.use_repstos, choices.exercise_floating_point, b->original.get()); + MaybeFlush(*b->original); + + MalignBuffer *head = b->original.get(); + + if (do_hashes) { + checksums.hash_value = choices.hasher->Hash(*head); + } + + if (do_compress) { + // Run our randomly chosen compressor. + if (!b->compressed) b->Alloc(&b->compressed); + b->compressed->Initialize(Alignment(), choices.buf_size); + MaybeFlush(*b->compressed); + + const auto s = zlib_.Compress(*head, b->compressed.get()); + if (!s.ok()) { + return ReturnError( + "Compression", + absl::StrCat(Json("syndrome", s.message()), ", ", writer_ident)); + } + MaybeFlush(*b->compressed); + head = b->compressed.get(); + } + + b->pre_encrypted = head; + if (do_encrypt) { + // Encrypt. + if (!b->encrypted) b->Alloc(&b->encrypted); + b->encrypted->Initialize(Alignment(), head->size()); + MaybeFlush(*b->encrypted); + if (choices.madvise) b->encrypted->MadviseDontNeed(); + + auto s = cpu_check::Crypto::Encrypt(*head, b->encrypted.get(), + &checksums.crypto_purse); + if (!s.ok()) { + return ReturnError(s.message(), writer_ident); + } + + MaybeFlush(*b->encrypted); + head = b->encrypted.get(); + } + + // Make a copy. + b->pre_copied = head; + if (!b->copied) b->Alloc(&b->copied); + b->copied->Initialize(Alignment(), head->size()); + MaybeFlush(*b->copied); + if (choices.madvise) b->copied->MadviseDontNeed(); + std::string syndrome = b->copied->CopyFrom(*head, choices.copy_method); + + if (!syndrome.empty()) { + return ReturnError( + "writer-detected-copy", + absl::StrCat(JsonRecord("syndrome", syndrome), ", ", writer_ident)); + } + MaybeFlush(*b->copied); + return checksums; +} + +absl::Status Worker::CheckComputations(const std::string &writer_reader_ident, + const Choices &choices, + const Checksums &checksums, + BufferSet *b) { + // Re-verify buffer copy + std::string syndrome = b->copied->Syndrome(*b->pre_copied); + if (!syndrome.empty()) { + return ReturnError("copy", absl::StrCat(JsonRecord("syndrome", syndrome), + ", ", writer_reader_ident)); + } + + MaybeFlush(*b->copied); + MalignBuffer *head = b->copied.get(); + + if (do_encrypt) { + // Decrypt. + if (!b->decrypted) b->Alloc(&b->decrypted); + b->decrypted->Initialize(Alignment(), head->size()); + MaybeFlush(*b->decrypted); + + if (choices.madvise) b->decrypted->MadviseDontNeed(); + auto s = cpu_check::Crypto::Decrypt(*head, checksums.crypto_purse, + b->decrypted.get()); + if (!s.ok()) { + return ReturnError(s.message(), writer_reader_ident); + } + + MaybeFlush(*b->decrypted); + head = b->decrypted.get(); + syndrome = b->pre_encrypted->Syndrome(*head); + if (!syndrome.empty()) { + return ReturnError("decryption_mismatch", + absl::StrCat(JsonRecord("syndrome", syndrome), ", ", + writer_reader_ident)); + } + } + + if (do_compress) { + // Run decompressor. + if (!b->decompressed) b->Alloc(&b->decompressed); + b->decompressed->Initialize(Alignment(), choices.buf_size); + MaybeFlush(*b->decompressed); + + if (choices.madvise) b->decompressed->MadviseDontNeed(); + const auto s = zlib_.Decompress(*head, b->decompressed.get()); + if (!s.ok()) { + return ReturnError("uncompression", + absl::StrCat(Json("syndrome", s.message()), ", ", + writer_reader_ident)); + } + if (b->decompressed->size() != choices.buf_size) { + std::stringstream ss; + ss << "dec_length: " << b->decompressed->size() + << " vs: " << choices.buf_size; + return ReturnError( + "decompressed_size", + absl::StrCat(Json("syndrome", ss.str()), ", ", writer_reader_ident)); + } + MaybeFlush(*b->decompressed); + head = b->decompressed.get(); + } + + if (!b->re_made) b->Alloc(&b->re_made); + b->re_made->Initialize(Alignment(), choices.buf_size); + const cpu_check::FloatingPointResults f_r = pattern_generators_.Generate( + *choices.pattern_generator, choices.hole, round_, choices.copy_method, + choices.use_repstos, choices.exercise_floating_point, b->re_made.get()); + syndrome = b->original->Syndrome(*b->re_made); + + if (!syndrome.empty()) { + return ReturnError("re-make", absl::StrCat(JsonRecord("syndrome", syndrome), + ", ", writer_reader_ident)); + } + + if (checksums.floating_point_results != f_r) { + std::stringstream ss; + ss << "Was: " << checksums.floating_point_results.d << " Is: " << f_r.d; + return ReturnError("fp-double", absl::StrCat(Json("syndrome", ss.str()), + ", ", writer_reader_ident)); + } + + if (do_hashes) { + // Re-run hash func. + const std::string hash = choices.hasher->Hash(*head); + if (checksums.hash_value != hash) { + std::stringstream ss; + ss << "hash was: " << checksums.hash_value << " is: " << hash; + return ReturnError("hash", absl::StrCat(Json("syndrome", ss.str()), ", ", + writer_reader_ident)); + } + } + + auto s = silkscreen_->CheckMySlots(tid_, round_); + if (!s.ok()) { + return ReturnError("Silkscreen", + absl::StrCat(s.message(), ", ", writer_reader_ident)); + } + return absl::OkStatus(); +} + +std::vector Worker::CheckerTids() { + constexpr int kCheckers = 2; + if (!do_hop) { + return std::vector(kCheckers, tid_); + } + std::vector candidates; + for (int i : tid_list_) { + if (i != tid_) candidates.push_back(i); + } + std::shuffle(candidates.begin(), candidates.end(), rndeng_); + std::vector v; + v.reserve(kCheckers); + for (int i = 0; i < kCheckers; i++) { + v.push_back(i < candidates.size() ? candidates[i] : tid_); + } + return v; } void Worker::Run() { - // Array of hash/checksum routines. - typedef struct { - const char *name; - std::string (*func)(const MalignBuffer &); - } hashItem; - std::vector hashers = { - { - "MD5", - [](const MalignBuffer &s) -> std::string { - return OpenSSL_Hash(s, EVP_md5()); - }, - }, - { - "SHA1", - [](const MalignBuffer &s) -> std::string { - return OpenSSL_Hash(s, EVP_sha1()); - }, - }, - { - "SHA256", - [](const MalignBuffer &s) -> std::string { - return OpenSSL_Hash(s, EVP_sha256()); - }, - }, - { - "SHA512", - [](const MalignBuffer &s) -> std::string { - return OpenSSL_Hash(s, EVP_sha512()); - }, - }, - { - "ADLER32", // exported by zlib - [](const MalignBuffer &s) -> std::string { - uLong c = adler32(0, Z_NULL, 0); - c = adler32(c, (const Bytef *)s.data(), s.size()); - return HexData((const char *)&c, sizeof(c)); - }, - }, - { - "CRC32", // exported by zlib. - [](const MalignBuffer &s) -> std::string { - uLong c = crc32(0, Z_NULL, 0); - c = crc32(c, (const Bytef *)s.data(), s.size()); - return HexData((const char *)&c, sizeof(c)); - }, - }, - { - "CRC32C", // crc32 instruction on SSSE3 - [](const MalignBuffer &s) -> std::string { - uint32_t c = crc32c(s.data(), s.size()); - return HexData((const char *)&c, sizeof(c)); - }, - }, - { - "FarmHash64", // Google farmhash - [](const MalignBuffer &s) -> std::string { - uint64_t c = util::Hash64(s.data(), s.size()); - return HexData((const char *)&c, sizeof(c)); - }, - }, - }; - - // Array of compression routines. - typedef struct { - const char *name; - int (*enc)(MalignBuffer *, const MalignBuffer &); - int (*dec)(MalignBuffer *, const MalignBuffer &); - } compressorItem; - std::vector compressors = { - { - "ZLIB", - [](MalignBuffer *o, const MalignBuffer &s) { - uLongf olen = compressBound(s.size()); - o->resize(olen); - int err = compress2((Bytef *)o->data(), &olen, (Bytef *)s.data(), - s.size(), Z_BEST_SPEED); - if (err != Z_OK) { - LOG(DEBUG) << "zlib compression failed: " << err - << " srclen: " << s.size() - << " destlen: " << o->size(); - return err; - } - o->resize(olen); - return 0; - }, - [](MalignBuffer *o, const MalignBuffer &s) { - uLongf olen = o->size(); - int err = uncompress((Bytef *)o->data(), &olen, (Bytef *)s.data(), - s.size()); - if (err != Z_OK) { - LOG(DEBUG) << "zlib decompression failed: " << err - << " srclen: " << s.size() - << " destlen: " << o->size(); - return err; - } - o->resize(olen); - return 0; - }, - }, - }; - - // Choose generator and compressor uniformly. - std::uniform_int_distribution gen_dist( - 0, do_provenance ? 0 : kGenerators.size() - 1); - auto Gen = std::bind(gen_dist, rndeng_); - std::uniform_int_distribution comp_dist(0, compressors.size() - 1); - auto Comp = std::bind(comp_dist, rndeng_); - - // Run one randomly-chosen hash routine each round. - size_t hash_choice; - std::string hash_value; - - EVP_CIPHER_CTX *cipher_ctx; - cipher_ctx = EVP_CIPHER_CTX_new(); + const double t0 = TimeInSeconds(); // Creates FVT controller if we can do so. if (do_fvt) { @@ -1354,22 +677,20 @@ void Worker::Run() { // MalignBuffers are allocated once if !do_madvise. Otherwise they are // reallocated each iteration of the main loop, creating much more memory // allocator work, which may itself exhibit, and suffer from, CPU defects. - std::unique_ptr b; + std::unique_ptr b = std::make_unique(); - uint64_t expected_slot_count = 0; - - Avx avx; - - while (!exiting) { + while (!stopper_->Expired()) { if (std::thread::hardware_concurrency() > 1) { if (!SetAffinity(tid_)) { LOG(WARN) << "Couldnt run on " << tid_ << " sleeping a bit"; - sleep(30); + stopper_->BoundedSleep(30); continue; } } round_++; - if (!b) { + + if (do_madvise) { + // Release and reallocate MalignBuffers. b.reset(new BufferSet); } @@ -1383,17 +704,13 @@ void Worker::Run() { fvt_controller_->MonitorFrequency(); } - auto Turbo = [&turbo_mhz](){ - return Json("turbo", turbo_mhz); - }; + auto Turbo = [&turbo_mhz]() { return Json("turbo", turbo_mhz); }; auto Tid = [this](int tid) { return "{ " + Json("tid", tid) + (do_fvt ? ", " + FVT() : "") + " }"; }; - auto Writer = [this, Tid](){ - return "\"writer\": " + Tid(tid_); - }; + auto Writer = [this, Tid]() { return "\"writer\": " + Tid(tid_); }; LOG_EVERY_N_SECS(INFO, 30) << Jstat( Json("elapsed_s", static_cast(TimeInSeconds() - t0)) + ", " + @@ -1404,395 +721,62 @@ void Worker::Run() { ", " + Json("maxFreq", fvt_controller_->max_mHz()) : "")); - if (do_avx_heavy) { - const std::string e = avx.MaybeGoHot(); - if (!e.empty()) { - LOG(ERROR) << Jfail(e, Writer() + ", " + Turbo()); - errorCount++; - } - } + const Choices choices = MakeChoices(b.get()); + const std::string writer_ident = + absl::StrCat(Writer(), ", ", Turbo(), ", ", choices.summary); -#ifdef USE_BORINGSSL - if (do_ssl_self_check && BORINGSSL_self_test() == 0) { - LOG(ERROR) << Jfail("BORINGSSL_self_test", Writer() + ", " + Turbo()); + auto s = DoComputations(writer_ident, choices, b.get()); + if (!s.ok()) { + LOG(ERROR) << s.status().message(); + LOG(ERROR) << Suspect(tid_); errorCount++; continue; } -#endif + const Checksums checksums = s.value(); - const bool madvise = - do_madvise && std::uniform_int_distribution(0, 1)(rndeng_); - - const size_t bufsize = - std::uniform_int_distribution(kBufMin, kBufMax)(rndeng_); - auto &gen = kGenerators[Gen()]; - auto &comp = compressors[Comp()]; - - const MalignBuffer::PunchedHole hole = PunchedHole(bufsize); - - const MalignBuffer::CopyMethod copy_method = CopyMethod(); - - const bool use_repstos = - do_repstosb && std::uniform_int_distribution(0, 1)(rndeng_); - - auto BlockSummary = [&]() { - std::stringstream block_summary; - block_summary - << Json("pattern", gen.name) << ", " - << Json("copy", MalignBuffer::ToString(copy_method)) << ", " - << Json("memset", use_repstos ? "rep;sto" : "memset") << ", " - << JsonBool("madvise", madvise) << ", " - << Json("size", bufsize) << ", " - << Json("pid", pid_) << ", " - << Json("round", round_) << ", " - << hole.ToString(); - return block_summary.str(); - }; - - auto WriterInfo = [&](){ - return Writer() + ", " + Turbo() + ", " + BlockSummary(); - }; - - if (round_ > 1) { - uint64_t slots_read = 0; - uint64_t errs_this_round = 0; - const uint64_t kErrLimit = 20; - for (size_t k = 0; k < Silkscreen::kSize; k++) { - const std::string err = silkscreen_->CheckMySlot(tid_, round_ - 1, k); - if (!err.empty()) { - slots_read++; - if (err != "=") { - errs_this_round++; - if (errs_this_round <= kErrLimit) { - errorCount++; - LOG(ERROR) << Jfail("Silkscreen", JsonRecord("syndrome", err) + - ", " + WriterInfo()); - } else { - // When Silkscreen fails, it often fails, on several bad machines, - // in a surprising way: all slots owned by reading tid are - // are corrupt, in a way that suggests the previous round's - // writes never happened. Weird, and deserves some study, but - // meanwhile the log spew is suppressed. - } - } - } + // Check the computation. Twice if the first check fails. + const std::vector checker_tids = CheckerTids(); + std::vector failing_tids; + for (int c : checker_tids) { + int newcpu = c; + if (!SetAffinity(newcpu)) { + // Tough luck, can't run on chosen CPU. + // Validate on same cpu we were on. + newcpu = tid_; } - if (errs_this_round > kErrLimit) { - LOG(ERROR) << Jfail( - "Silkscreen", - JsonRecord("syndrome", Json("many_errors", errs_this_round) + ", " + - Json("slots_read", slots_read)) + - ", " + WriterInfo()); - errorCount++; - } - if (expected_slot_count != slots_read) { - LOG(ERROR) << Jfail("Silkscreen", - Json("read", slots_read) + ", " + - Json("expected", expected_slot_count) + ", " + - WriterInfo()); + + auto Reader = [&Tid, &newcpu]() { return "\"reader\": " + Tid(newcpu); }; + const std::string writer_reader_ident = absl::StrCat( + Writer(), ", ", Reader(), ", ", Turbo(), ", ", choices.summary); + + const absl::Status check_status = + CheckComputations(writer_reader_ident, choices, checksums, b.get()); + if (check_status.ok()) { + // It suffices to check just once if the checker confirms that + // computation was correct. + break; + } else { + failing_tids.push_back(newcpu); + LOG(ERROR) << check_status.message(); errorCount++; } } - const uint64_t slots_written = silkscreen_->WriteMySlots(tid_, round_); - if (!expected_slot_count) { - expected_slot_count = slots_written; - } - if (expected_slot_count != slots_written) { - LOG(ERROR) << Jfail("Silkscreen", - Json("written", slots_written) + ", " + - Json("expected", expected_slot_count) + ", " + - WriterInfo()); - errorCount++; - } - - if (do_avx_heavy) { - // If we tried to do AVX heavy stuff. Try to run AVX heavy again to try - // to spike current. - const std::string e = avx.BurnIfAvxHeavy(); - if (!e.empty()) { - LOG(ERROR) << Jfail(e, Writer() + ", " + Turbo()); - errorCount++; + if (!failing_tids.empty()) { + // Guess which LPU is the most likely culprit. The guess is pretty good + // for low failure rate LPUs that haven't corrupted crucial common state. + if (failing_tids.size() > 1) { + // Both checkers think the computation was wrong, likely culprit is the + // writer. + LOG(ERROR) << Suspect(tid_); + } else { + // Only one checker thinks the computation was wrong. Likely he's the + // culprit since the other checker and the writer agree. + LOG(ERROR) << Suspect(failing_tids[0]); } - } - - const auto buffer_seed = rndeng_(); - if (!b->original) b->Alloc(&b->original); - b->original->Initialize(Alignment(), bufsize); - const FloatingPointResults f = - GenerateData(gen, hole, buffer_seed, - copy_method, use_repstos, b->original.get()); - MaybeFlush(*b->original); - - MalignBuffer* head = b->original.get(); - - if (do_hashes) { - hash_choice = - std::uniform_int_distribution(0, hashers.size() - 1)(rndeng_); - hash_value = hashers[hash_choice].func(*head); - } - - if (do_compress) { - // Run our randomly chosen compressor. - if (!b->compressed) b->Alloc(&b->compressed); - b->compressed->Initialize(Alignment(), bufsize); - MaybeFlush(*b->compressed); - - const int err = comp.enc(b->compressed.get(), *head); - LOG(DEBUG) << WriterInfo() - << " original->size(): " << head->size() - << ", compressed.size(): " << b->compressed->size() << "."; - if (err) { - LOG(ERROR) << Jfail("Compression", - Json("syndrome", err) + ", " + WriterInfo()); - errorCount++; - continue; - } - MaybeFlush(*b->compressed); - head = b->compressed.get(); - LOG(DEBUG) << WriterInfo() << "compress done."; - if (exiting) break; - } - - const unsigned char key[33] = "0123456789abcdef0123456789abcdef"; - const std::string ivec(b->original->data(), kBufMin); - unsigned char gmac[16]; - - const MalignBuffer* const unencrypted = head; - if (do_encrypt) { - // Encrypt. - if (!b->encrypted) b->Alloc(&b->encrypted); - b->encrypted->Initialize(Alignment(), head->size()); - MaybeFlush(*b->encrypted); - int enc_len = 0, enc_unused_len = 0; - EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key, - (unsigned char *)ivec.data(), 1); - - if (madvise) MadviseDontNeed(*b->encrypted); - if (EVP_CipherUpdate( - cipher_ctx, (unsigned char *)b->encrypted->data(), &enc_len, - (unsigned char *)head->data(), head->size()) != 1) { - LOG(ERROR) << Jfail("EVP_CipherUpdate", WriterInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - if (EVP_CipherFinal_ex(cipher_ctx, nullptr, &enc_unused_len) != 1) { - LOG(ERROR) << Jfail("encrypt_EVP_CipherFinal_ex", WriterInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - enc_len += enc_unused_len; - if (enc_len != (int)b->encrypted->size()) { - std::stringstream ss; - ss << "enc_length: " << enc_len << " vs: " << b->encrypted->size(); - LOG(ERROR) << Jfail("encrypt_length_mismatch", - Json("syndrome", ss.str()) + ", " + WriterInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_GET_TAG, sizeof(gmac), - gmac) != 1) { - LOG(ERROR) << Jfail("EVP_CTRL_GCM_GET_TAG", WriterInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - EVP_CIPHER_CTX_cleanup(cipher_ctx); - MaybeFlush(*b->encrypted); - head = b->encrypted.get(); - LOG(DEBUG) << "Encrypt done " << WriterInfo(); - if (exiting) break; - } - - // Make a copy. - if (!b->copied) b->Alloc(&b->copied); - b->copied->Initialize(Alignment(), head->size()); - MaybeFlush(*b->copied); - if (madvise) MadviseDontNeed(*b->copied); - std::string syndrome = b->copied->CopyFrom(*head, copy_method); - - if (!syndrome.empty()) { - LOG(ERROR) << Jfail( - "writer-detected-copy", - JsonRecord("syndrome", syndrome) + ", " + WriterInfo()); - errorCount++; continue; } - MaybeFlush(*b->copied); - - // Switch to an alternate CPU. - - int newcpu = tid_; - if (do_hop && std::thread::hardware_concurrency() > 1) { - std::vector cpus; - for (int i : tid_list_) { - if (i == tid_) continue; - cpus.push_back(i); - } - if (!cpus.empty()) { - int cpuoff = - std::uniform_int_distribution(0, cpus.size() - 1)(rndeng_); - newcpu = cpus[cpuoff]; - cpus.erase(cpus.begin() + cpuoff); - if (!SetAffinity(newcpu)) { - // Tough luck, can't run on chosen CPU. - // Validate on same cpu we were on. - newcpu = tid_; - } - } - } - - auto Reader = [&Tid, &newcpu](){ - return "\"reader\": " + Tid(newcpu); - }; - auto WriterReaderInfo = [&](){ - return - Writer() + ", " + Reader() + ", " + Turbo() + ", " + BlockSummary(); - }; - - // Re-verify buffer copy - syndrome = b->copied->Syndrome(*head); - if (!syndrome.empty()) { - LOG(ERROR) << Jfail( - "copy", JsonRecord("syndrome", syndrome) + ", " + WriterInfo()); - errorCount++; - continue; - } - - MaybeFlush(*b->copied); - head = b->copied.get(); - - if (do_encrypt) { - // Decrypt. - if (!b->decrypted) b->Alloc(&b->decrypted); - b->decrypted->Initialize(Alignment(), head->size()); - MaybeFlush(*b->decrypted); - - int dec_len = 0, dec_extra_len = 0; - EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key, - (unsigned char *)ivec.data(), 0); - if (madvise) MadviseDontNeed(*b->decrypted); - if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_SET_TAG, sizeof(gmac), - gmac) != 1) { - LOG(ERROR) << Jfail("EVP_CTRL_GCM_SET_TAG", WriterReaderInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - if (EVP_CipherUpdate( - cipher_ctx, (unsigned char *)b->decrypted->data(), &dec_len, - (unsigned char *)head->data(), head->size()) != 1) { - LOG(ERROR) << Jfail("decryption", WriterReaderInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - if (EVP_CipherFinal_ex( - cipher_ctx, (unsigned char *)(b->decrypted->data() + dec_len), - &dec_extra_len) != 1) { - LOG(ERROR) << Jfail("decrypt_EVP_CipherFinal_ex", WriterReaderInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - dec_len += dec_extra_len; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - if (dec_len != (int)b->decrypted->size()) { - std::stringstream ss; - ss << "dec_length: " << dec_len << " vs: " << b->decrypted->size(); - LOG(ERROR) << Jfail("decrypt_length_mismatch", - Json("syndrome", ss.str()) + ", " + - WriterReaderInfo()); - errorCount++; - continue; - } - MaybeFlush(*b->decrypted); - head = b->decrypted.get(); - syndrome = unencrypted->Syndrome(*head); - if (!syndrome.empty()) { - LOG(ERROR) << Jfail( - "decryption_mismatch", - JsonRecord("syndrome", syndrome) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - LOG(DEBUG) << WriterReaderInfo() << " decrypt done"; - if (exiting) break; - } - - if (do_compress) { - // Run decompressor. - if (!b->decompressed) b->Alloc(&b->decompressed); - b->decompressed->Initialize(Alignment(), bufsize); - MaybeFlush(*b->decompressed); - - if (madvise) MadviseDontNeed(*b->decompressed); - const int err = comp.dec(b->decompressed.get(), *head); - if (err) { - LOG(ERROR) << Jfail("uncompression", - Json("syndrome", err) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - if (b->decompressed->size() != bufsize) { - std::stringstream ss; - ss << "dec_length: " << b->decompressed->size() << " vs: " << bufsize; - LOG(ERROR) << Jfail( - "decompressed_size", - Json("syndrome", ss.str()) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - MaybeFlush(*b->decompressed); - head = b->decompressed.get(); - LOG(DEBUG) << WriterReaderInfo() << " uncompress done"; - } - - if (!b->re_made) b->Alloc(&b->re_made); - b->re_made->Initialize(Alignment(), bufsize); - const FloatingPointResults f_r = - GenerateData(gen, hole, buffer_seed, - copy_method, use_repstos, b->re_made.get()); - syndrome = b->original->Syndrome(*b->re_made); - - if (!syndrome.empty()) { - LOG(ERROR) << Jfail("re-make", JsonRecord("syndrome", syndrome) + ", " + - WriterReaderInfo()); - errorCount++; - continue; - } - - if (f != f_r) { - std::stringstream ss; - ss << "Was: " << f.d << " Is: " << f_r.d; - LOG(ERROR) << Jfail( - "fp-double", Json("syndrome", ss.str()) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - - if (do_hashes) { - // Re-run hash func. - std::string hash = hashers[hash_choice].func(*head); - if (hash_value != hash) { - std::stringstream ss; - ss << "hash was: " << hash_value << " is: " << hash; - LOG(ERROR) << Jfail( - "hash", Json("syndrome", ss.str()) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - LOG(DEBUG) << WriterReaderInfo() << " rehash done"; - } - - // Release MalignBuffer memory allocations. - if (do_madvise) b.reset(nullptr); - successCount++; } - EVP_CIPHER_CTX_free(cipher_ctx); LOG(INFO) << "tid " << tid_ << " exiting."; } @@ -1833,12 +817,10 @@ int main(int argc, char **argv) { std::vector tid_list; int64_t timeout = 0; -#ifdef IN_GOOGLE3 // Initialize the symbolizer to get a human-readable stack trace. absl::InitializeSymbolizer(argv[0]); absl::FailureSignalHandlerOptions options; absl::InstallFailureSignalHandler(options); -#endif for (int i = 1; i < argc; i++) { const char *flag = argv[i]; @@ -1851,39 +833,35 @@ int main(int argc, char **argv) { case 'b': do_ssl_self_check = false; break; - case 'c': - { - std::string c = ""; - for (flag++; *flag != 0; flag++) { - c += *flag; - } - std::stringstream s(c); - int t; - while (s >> t) { - tid_list.push_back(t); - if (s.peek() == ',') s.ignore(); - } + case 'c': { + std::string c = ""; + for (flag++; *flag != 0; flag++) { + c += *flag; } - break; + std::stringstream s(c); + int t; + while (s >> t) { + tid_list.push_back(t); + if (s.peek() == ',') s.ignore(); + } + } break; case 'd': do_repstosb = false; break; case 'e': do_encrypt = false; break; - case 'f': - { - std::string c(++flag); - flag += c.length(); - std::stringstream s(c); - s >> fixed_min_frequency; - fixed_max_frequency = fixed_min_frequency; - if (s.get() == '-') { - s >> fixed_max_frequency; - do_freq_sweep = true; - } + case 'f': { + std::string c(++flag); + flag += c.length(); + std::stringstream s(c); + s >> fixed_min_frequency; + fixed_max_frequency = fixed_min_frequency; + if (s.get() == '-') { + s >> fixed_max_frequency; + do_freq_sweep = true; } - break; + } break; case 'F': do_flush = true; break; @@ -1920,14 +898,12 @@ int main(int argc, char **argv) { do_hashes = false; do_compress = false; break; - case 'q': - { - std::string c(++flag); - flag += c.length(); - std::stringstream s(c); - s >> error_limit; - } - break; + case 'q': { + std::string c(++flag); + flag += c.length(); + std::stringstream s(c); + s >> error_limit; + } break; case 'r': do_repmovsb = false; break; @@ -1946,22 +922,18 @@ int main(int argc, char **argv) { case 'Y': do_freq_sweep = true; break; - case 'k': - { - std::string c(++flag); - flag += c.length(); - std::stringstream s(c); - s >> seconds_per_freq; - } - break; - case 't': - { - std::string c(++flag); - flag += c.length(); - std::stringstream s(c); - s >> timeout; - } - break; + case 'k': { + std::string c(++flag); + flag += c.length(); + std::stringstream s(c); + s >> seconds_per_freq; + } break; + case 't': { + std::string c(++flag); + flag += c.length(); + std::stringstream s(c); + s >> timeout; + } break; case 'z': do_compress = false; break; @@ -1995,11 +967,7 @@ int main(int argc, char **argv) { std::vector threads; std::vector workers; - std::vector words = ReadDict(); - if (words.empty()) { - LOG(ERROR) << "No word list found."; - exit(1); - } + int cpus = std::thread::hardware_concurrency(); LOG(INFO) << "Detected hardware concurrency: " << cpus; @@ -2024,26 +992,28 @@ int main(int argc, char **argv) { } } + const double t0 = TimeInSeconds(); // Silkscreen instance shared by all threads. - Silkscreen silkscreen(tid_list); + cpu_check::Silkscreen silkscreen(tid_list); + + static Stopper stopper(timeout); // Shared by all threads for (int tid : tid_list) { - workers.push_back(new Worker(getpid(), &words, tid_list, tid, &silkscreen)); + workers.push_back( + new Worker(getpid(), tid_list, tid, &silkscreen, &stopper)); threads.push_back(new std::thread(&Worker::Run, workers.back())); } - signal(SIGTERM, [](int) { exiting = true; }); - signal(SIGINT, [](int) { exiting = true; }); + + signal(SIGTERM, [](int) { stopper.Stop(); }); + signal(SIGINT, [](int) { stopper.Stop(); }); struct timeval last_cpu = {0, 0}; double last_time = t0; - while (!exiting) { - sleep(60); + while (!stopper.Expired()) { + stopper.BoundedSleep(60); struct rusage ru; double secs = TimeInSeconds(); - if (timeout > 0 && secs >= t0 + timeout) { - exiting = true; - } double secondsPerError = (secs - t0) / errorCount.load(); if (getrusage(RUSAGE_SELF, &ru) == -1) { LOG(ERROR) << "getrusage failed: " << strerror(errno); @@ -2052,8 +1022,8 @@ int main(int argc, char **argv) { (ru.ru_utime.tv_usec - last_cpu.tv_usec)) / 1000000.0; LOG(INFO) << "Errors: " << errorCount.load() - << " Successes: " << successCount.load() - << " CPU " << cpu / (secs - last_time) << " s/s" + << " Successes: " << successCount.load() << " CPU " + << cpu / (secs - last_time) << " s/s" << " Seconds Per Error: " << secondsPerError; last_cpu = ru.ru_utime; } diff --git a/crypto.cc b/crypto.cc new file mode 100644 index 0000000..c8dd282 --- /dev/null +++ b/crypto.cc @@ -0,0 +1,112 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "crypto.h" + +#include "config.h" +#include "absl/status/status.h" + +namespace cpu_check { + +namespace { +constexpr unsigned char key[33] = "0123456789abcdef0123456789abcdef"; +}; // namespace + +absl::Status Crypto::Encrypt(const MalignBuffer &plain_text, + MalignBuffer *cipher_text, CryptoPurse *purse) { + memset(purse->i_vec, 0, sizeof(purse->i_vec)); + memcpy(purse->i_vec, plain_text.data(), + std::min(plain_text.size(), sizeof(purse->i_vec))); + + int enc_len = 0; + int enc_unused_len = 0; + EVP_CIPHER_CTX *cipher_ctx = EVP_CIPHER_CTX_new(); + + EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key, purse->i_vec, 1); + if (EVP_CipherUpdate( + cipher_ctx, + reinterpret_cast(cipher_text->data()), + &enc_len, reinterpret_cast(plain_text.data()), + plain_text.size()) != 1) { + return ReturnError("EVP_CipherUpdate", cipher_ctx); + } + if (EVP_CipherFinal_ex(cipher_ctx, nullptr, &enc_unused_len) != 1) { + return ReturnError("encrypt_EVP_CipherFinal_ex", cipher_ctx); + } + enc_len += enc_unused_len; + if (enc_len != (int)cipher_text->size()) { + return ReturnError("encrypt_length_mismatch", cipher_ctx); + } + if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_GET_TAG, + sizeof(purse->gmac_tag), purse->gmac_tag) != 1) { + return ReturnError("EVP_CTRL_GCM_GET_TAG", cipher_ctx); + } + EVP_CIPHER_CTX_free(cipher_ctx); + return absl::OkStatus(); +} + +absl::Status Crypto::Decrypt(const MalignBuffer &cipher_text, + const CryptoPurse &purse, + MalignBuffer *plain_text) { + int dec_len = 0; + int dec_extra_len = 0; + EVP_CIPHER_CTX *cipher_ctx = EVP_CIPHER_CTX_new(); + + EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key, purse.i_vec, 0); + + // Make a non-const copy of gmac_tag because that's what EVP_CIPHER_CTX_ctrl + // requires, even though it won't be modified in this use. + unsigned char copied_tag[sizeof(purse.gmac_tag)]; + memcpy(copied_tag, purse.gmac_tag, sizeof(purse.gmac_tag)); + + if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_SET_TAG, sizeof(copied_tag), + reinterpret_cast(copied_tag)) != 1) { + return ReturnError("EVP_CTRL_GCM_SET_TAG", cipher_ctx); + } + if (EVP_CipherUpdate( + cipher_ctx, reinterpret_cast(plain_text->data()), + &dec_len, reinterpret_cast(cipher_text.data()), + cipher_text.size()) != 1) { + return ReturnError("Decryption", cipher_ctx); + } + if (EVP_CipherFinal_ex( + cipher_ctx, + reinterpret_cast(plain_text->data() + dec_len), + &dec_extra_len) != 1) { + return ReturnError("decrypt_EVP_CipherFinal_ex", cipher_ctx); + } + dec_len += dec_extra_len; + if (dec_len != (int)plain_text->size()) { + return ReturnError("decrypt_length_mismatch", cipher_ctx); + } + EVP_CIPHER_CTX_free(cipher_ctx); + return absl::OkStatus(); +} + +absl::Status Crypto::SelfTest() { +#ifdef USE_BORINGSSL + if (BORINGSSL_self_test() == 0) { + return absl::Status(absl::StatusCode::kInternal, "BORINGSSL_self_test"); + } +#endif + + return absl::OkStatus(); +} + +absl::Status Crypto::ReturnError(absl::string_view message, + EVP_CIPHER_CTX *cipher_ctx) { + EVP_CIPHER_CTX_free(cipher_ctx); + return absl::Status(absl::StatusCode::kInternal, message); +} +}; // namespace cpu_check diff --git a/crypto.h b/crypto.h new file mode 100644 index 0000000..f92c5d4 --- /dev/null +++ b/crypto.h @@ -0,0 +1,54 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_CRYPTO_H_ +#define THIRD_PARTY_CPU_CHECK_CRYPTO_H_ + +#include "malign_buffer.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include +#include + +namespace cpu_check { + +class Crypto { + public: + // Encryption produces these values, which are consumed by decryption. + struct CryptoPurse { + unsigned char i_vec[12]; + unsigned char gmac_tag[16]; + }; + + // Encrypts 'plain_text' to 'cipher_text' and stores i_vec and gmac + // in 'purse'. + static absl::Status Encrypt(const MalignBuffer &plain_text, + MalignBuffer *cipher_text, CryptoPurse *purse); + + // Decrypts 'cipher_text' into 'plain_text' using i_vec and gmac from 'purse'. + static absl::Status Decrypt(const MalignBuffer &cipher_text, + const CryptoPurse &purse, + MalignBuffer *plain_text); + + // Runs crypto self test, if available. + static absl::Status SelfTest(); + + private: + // Returns kInternal error and frees context 'cipher_ctx'. + static absl::Status ReturnError(absl::string_view message, + EVP_CIPHER_CTX *cipher_ctx); +}; + +}; // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_CRYPTO_H_ diff --git a/hasher.cc b/hasher.cc new file mode 100644 index 0000000..fc25cf4 --- /dev/null +++ b/hasher.cc @@ -0,0 +1,95 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hasher.h" + +#include "crc32c.h" +#include "utils.h" +#include "third_party/farmhash/src/farmhash.h" +#include +#include +#include + +namespace cpu_check { +namespace { +std::string OpenSSL_Hash(const MalignBuffer &s, const EVP_MD *type) { + EVP_MD_CTX *ctx; + ctx = EVP_MD_CTX_create(); + EVP_DigestInit_ex(ctx, type, nullptr); + std::string hash; + hash.resize(EVP_MD_CTX_size(ctx)); + MalignBuffer::InitializeMemoryForSanitizer(hash.data(), EVP_MD_CTX_size(ctx)); + EVP_DigestUpdate(ctx, s.data(), s.size()); + EVP_DigestFinal_ex(ctx, (uint8_t *)&hash[0], nullptr); + EVP_MD_CTX_destroy(ctx); + return HexStr(hash); +} +} // namespace + +std::string Md5::Hash(const MalignBuffer &b) const { + return OpenSSL_Hash(b, EVP_md5()); +} + +std::string Sha1::Hash(const MalignBuffer &b) const { + return OpenSSL_Hash(b, EVP_sha1()); +} + +std::string Sha256::Hash(const MalignBuffer &b) const { + return OpenSSL_Hash(b, EVP_sha256()); +} + +std::string Sha512::Hash(const MalignBuffer &b) const { + return OpenSSL_Hash(b, EVP_sha512()); +} + +std::string Adler32::Hash(const MalignBuffer &b) const { + uLong c = adler32(0, Z_NULL, 0); + c = adler32(c, reinterpret_cast(b.data()), b.size()); + return HexData(reinterpret_cast(&c), sizeof(c)); +} + +std::string Crc32::Hash(const MalignBuffer &b) const { + uLong c = crc32(0, Z_NULL, 0); + c = crc32(c, reinterpret_cast(b.data()), b.size()); + return HexData(reinterpret_cast(&c), sizeof(c)); +} + +std::string Crc32C::Hash(const MalignBuffer &b) const { + const uint32_t c = crc32c(b.data(), b.size()); + return HexData(reinterpret_cast(&c), sizeof(c)); +} + +std::string FarmHash64::Hash(const MalignBuffer &b) const { + const uint64_t c = util::Hash64(b.data(), b.size()); + return HexData(reinterpret_cast(&c), sizeof(c)); +} + +Hashers::Hashers() { + hashers_.emplace_back(new Md5); + hashers_.emplace_back(new Sha1); + hashers_.emplace_back(new Sha256); + hashers_.emplace_back(new Sha512); + hashers_.emplace_back(new Adler32); + hashers_.emplace_back(new Crc32); + hashers_.emplace_back(new Crc32C); + hashers_.emplace_back(new FarmHash64); +} + +const Hasher &Hashers::RandomHasher(uint64_t seed) const { + std::knuth_b rng(seed); + const size_t k = + std::uniform_int_distribution(0, hashers_.size() - 1)(rng); + return *hashers_[k]; +} +} // namespace cpu_check diff --git a/hasher.h b/hasher.h new file mode 100644 index 0000000..8d7ff53 --- /dev/null +++ b/hasher.h @@ -0,0 +1,96 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_HASH_H_ +#define THIRD_PARTY_CPU_CHECK_HASH_H_ + +#include +#include +#include + +#include "malign_buffer.h" + +namespace cpu_check { + +class Hasher { + public: + virtual ~Hasher() {} + virtual std::string Name() const = 0; + virtual std::string Hash(const MalignBuffer &b) const = 0; +}; + +class Md5 : public Hasher { + public: + std::string Name() const override { return "MD5"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Sha1 : public Hasher { + public: + std::string Name() const override { return "SHA1"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Sha256 : public Hasher { + public: + std::string Name() const override { return "SHA256"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Sha512 : public Hasher { + public: + std::string Name() const override { return "SHA512"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Adler32 : public Hasher { + public: + std::string Name() const override { return "ADLER32"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Crc32 : public Hasher { + public: + std::string Name() const override { return "CRC32"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Crc32C : public Hasher { + public: + std::string Name() const override { return "CRC32C"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class FarmHash64 : public Hasher { + public: + std::string Name() const override { return "FarmHash64"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Hashers { + public: + Hashers(); + + // Returns a randomly selected hasher. + const Hasher &RandomHasher(uint64_t seed) const; + + const std::vector> &hashers() const { + return hashers_; + } + + private: + std::vector> hashers_; +}; +} // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_HASH_H_ diff --git a/malign_buffer.cc b/malign_buffer.cc new file mode 100644 index 0000000..5845ffb --- /dev/null +++ b/malign_buffer.cc @@ -0,0 +1,382 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "malign_buffer.h" + +#include + +#if defined(__i386__) || defined(__x86_64__) +#include +#endif + +#include +#include + +#include +#include +#include + +#include "log.h" +#include "utils.h" + +#undef HAS_FEATURE_MEMORY_SANITIZER +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#define HAS_FEATURE_MEMORY_SANITIZER +#endif +#endif + +#if defined(__i386__) || defined(__x86_64__) +#define X86_TARGET_ATTRIBUTE(s) __attribute__((target(s))) +#else +#define X86_TARGET_ATTRIBUTE(s) +#endif + +namespace cpu_check { +namespace { + +inline void __movsb(char *dst, const char *src, size_t size) { +#if defined(__i386__) || defined(__x86_64__) + __asm__ __volatile__("rep movsb" + : "+D"(dst), "+S"(src), "+c"(size) + : + : "memory"); +#else + LOG(FATAL) << "Cannot rep;movsb"; +#endif +} + +inline void __stosb(void *dst, unsigned char c, size_t size) { +#if defined(__i386__) || defined(__x86_64__) + __asm__ __volatile__("rep stosb" : "+D"(dst), "+c"(size) : "a"(c) : "memory"); +#else + LOG(FATAL) << "Cannot rep;stosb"; +#endif +} + +inline void __sse_128_memcpy(char *dst, const char *src, size_t size) { +#if (defined(__i386__) || defined(__x86_64__)) + size_t blks = size / 16; + for (int i = 0; i < blks; i++) { + _mm_storeu_si128( + reinterpret_cast<__m128i *>(dst) + i, + _mm_loadu_si128(reinterpret_cast(src) + i)); + } + memcpy(dst + blks * 16, src + blks * 16, size - blks * 16); +#else + LOG(FATAL) << "SSE not available"; +#endif +} + +X86_TARGET_ATTRIBUTE("avx") +inline void __avx_256_memcpy(char *dst, const char *src, size_t size) { +#if (defined(__i386__) || defined(__x86_64__)) + size_t blks = size / 32; + for (int i = 0; i < blks; i++) { + _mm256_storeu_si256( + reinterpret_cast<__m256i *>(dst) + i, + _mm256_loadu_si256(reinterpret_cast(src) + i)); + } + memcpy(dst + blks * 32, src + blks * 32, size - blks * 32); +#else + LOG(FATAL) << "x86 only"; +#endif +} + +X86_TARGET_ATTRIBUTE("avx512f") +inline void __avx_512_memcpy(char *dst, const char *src, size_t size) { +#if (defined(__i386__) || defined(__x86_64__)) + size_t blks = size / 64; + for (int i = 0; i < blks; i++) { + _mm512_storeu_si512( + reinterpret_cast<__m512i *>(dst) + i, + _mm512_loadu_si512(reinterpret_cast(src) + i)); + } + memcpy(dst + blks * 64, src + blks * 64, size - blks * 64); +#else + LOG(FATAL) << "x86 only"; +#endif +} +} // namespace + +size_t MalignBuffer::RoundUpToPageSize(size_t k) { + return ((k + kPageSize - 1) / kPageSize) * kPageSize; +} + +// Helper to make MSAN happy. NOP if memory sanitizer is not enabled. +void MalignBuffer::InitializeMemoryForSanitizer(char *addr, size_t size) { +#ifdef HAS_FEATURE_MEMORY_SANITIZER + std::default_random_engine rnd; + std::uniform_int_distribution dist(std::numeric_limits::min(), + std::numeric_limits::max()); + for (size_t i = 0; i < size; i++) { + addr[i] = dist(rnd); + } +#endif +} + +const size_t MalignBuffer::kPageSize = sysconf(_SC_PAGESIZE); +const size_t MalignBuffer::kCacheLineSize = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); + +std::string MalignBuffer::ToString(CopyMethod m) { + switch (m) { + case kMemcpy: + return "memcpy"; + case kRepMov: + return "rep;mov"; + case kSseBy128: + return "sse:128"; + case kAvxBy256: + return "avx:256"; + case kAvxBy512: + return "avx:512"; + } +} + +size_t MalignBuffer::RandomAlignment(uint64_t seed) { + std::knuth_b rng(seed); + return std::uniform_int_distribution(0, kPageSize - 1)(rng); +} + +MalignBuffer::MalignBuffer(size_t capacity) + : capacity_(capacity), + base_address_( + aligned_alloc(kPageSize, RoundUpToPageSize(capacity) + kPageSize)) { + if (base_address_ == nullptr) { + LOG(FATAL) << "Failed allocate for capacity: " << capacity; + } + // There are lots of places that use unitialized MalignBuffer. So just + // fill some pseudo-random bytes if cpu_check is compiled with msan. + InitializeMemoryForSanitizer(static_cast(base_address_), capacity_); +} +MalignBuffer::MalignBuffer(size_t alignment_offset, absl::string_view s) + : MalignBuffer(s.size() + alignment_offset) { + Initialize(alignment_offset, s.size()); + CopyFrom(s, kMemcpy); +} + +MalignBuffer::~MalignBuffer() { free(base_address_); } + +void MalignBuffer::Initialize(size_t alignment_offset, size_t length) { + if (length > capacity_) { + LOG(FATAL) << "Length: " << length << " Capacity: " << capacity_; + } + if (alignment_offset >= kPageSize) { + LOG(FATAL) << "Alignment: " << alignment_offset + << " PageSize: " << kPageSize; + } + alignment_offset_ = alignment_offset; + length_ = length; + buffer_address_ = static_cast(base_address_) + alignment_offset_; +} + +void MalignBuffer::resize(size_t length) { + Initialize(alignment_offset_, length); +} + +std::string MalignBuffer::CopyFrom(const MalignBuffer &that, CopyMethod m) { + CopyFrom(absl::string_view(that.data(), that.size()), m); + return Syndrome(that); +} + +void MalignBuffer::CopyFrom(absl::string_view src, CopyMethod m) { + if (size() != src.size()) { + LOG(FATAL) << "this.size: " << size() << " src.size:" << src.size(); + } + CopyFrom(0, src, m); +} + +void MalignBuffer::CopyFrom(size_t pos, absl::string_view src, CopyMethod m) { + if (pos + src.size() > size()) { + LOG(FATAL) << "this.size: " << size() << " src.size:" << src.size() + << " pos: " << pos; + } + switch (m) { + case kMemcpy: + // Assumes memcpy doesn't use rep;movsb; false in lots of environments. + memcpy(data() + pos, src.data(), src.size()); + break; + case kRepMov: + __movsb(data() + pos, src.data(), src.size()); + break; + case kSseBy128: + __sse_128_memcpy(data() + pos, src.data(), src.size()); + break; + case kAvxBy256: + __avx_256_memcpy(data() + pos, src.data(), src.size()); + break; + case kAvxBy512: + __avx_512_memcpy(data() + pos, src.data(), src.size()); + break; + } +} + +std::string MalignBuffer::Syndrome(const MalignBuffer &that) const { + std::stringstream s; + std::string syndrome = CorruptionSyndrome(that); + if (syndrome.empty()) return ""; + s << syndrome << ", \"this\": \"" << static_cast(data()) + << "\", " + << "\"that\": \"" << static_cast(that.data()) << "\""; + return s.str(); +} + +std::string MalignBuffer::CorruptionSyndrome(const MalignBuffer &that) const { + std::stringstream s; + if (size() != that.size()) { + s << Json("unequalSizeThis", size()) << ", " + << Json("unequalSizeThat", that.size()); + return s.str(); + } + bool failed_memcmp = memcmp(data(), that.data(), that.size()); + + int wrong_bytes = 0; + int wrong_bits = 0; + int byte_faults = 0; + int first_wrong = INT_MAX; + int last_wrong = INT_MIN; + std::vector lane_errors(8, 0); + for (size_t i = 0; i < size(); i++) { + unsigned char a = *(data() + i); + unsigned char b = *(that.data() + i); + unsigned char d = a ^ b; + if (d) { + first_wrong = std::min(first_wrong, i); + last_wrong = std::max(last_wrong, i); + byte_faults |= d; + wrong_bytes++; + wrong_bits += __builtin_popcount(d); + for (size_t i = 0; i < 8; i++) { + if ((d >> i) & 1) { + lane_errors[i]++; + } + } + } + } + if (wrong_bits || wrong_bytes) { + const int range_width = (last_wrong - first_wrong) + 1; + s << Json("cmpResult", + (failed_memcmp ? "Failed_Memcmp" : "**Passed_Memcmp**")) + << ", " << Json("wrongByteCount", wrong_bytes) << ", " + << Json("wrongBitCount", wrong_bits) << ", " + << Json("corruptionWidth", range_width) << ", " + << Json("corruptStart", first_wrong) << ", " + << Json("corruptByteBitMask", byte_faults) << ", " + << "\"byBitLane\": ["; + for (size_t i = 0; i < 8; i++) { + if (i) s << ", "; + s << lane_errors[i]; + } + s << " ] "; + // Dump up to 64 corrupted locations. + std::stringstream dump; + dump << " \"byteErrors\": [ " << std::hex; + uint64_t buf_a = 0; + uint64_t buf_b = 0; + for (size_t k = 0; k < std::min(64, range_width); k++) { + uint8_t a = *(data() + first_wrong + k); + uint8_t b = *(that.data() + first_wrong + k); + if (k) dump << ", "; + dump << "[ " << std::setw(2) << "\"0x" << static_cast(a) << "\", " + << std::setw(2) << "\"0x" << static_cast(b) << "\" "; + buf_a = (buf_a >> 8) | static_cast(a) << 56; + buf_b = (buf_b >> 8) | static_cast(b) << 56; + if ((k >= 7) && (7 == ((first_wrong + k) % 8))) { + dump << ", " << CrackId(buf_a) << ", " << CrackId(buf_b); + buf_a = 0; + buf_b = 0; + } + dump << " ]"; + } + dump << " ] "; + return s.str() + ", " + dump.str(); + } else { + if (!failed_memcmp) return ""; + return Json("cmpResult", "**Failed_Memcmp**"); + } +} + +std::string MalignBuffer::CrackId(uint64_t v) const { + std::stringstream s; + s << std::hex << " [\"0x" << std::setw(4) << (v >> 48) << "\", \"0x" + << std::setw(6) << ((v >> 24) & 0xffffff) << "\", \"0x" << std::setw(6) + << (v & 0xffffff) << "\"]"; + return s.str(); +} + +void MalignBuffer::RandomFlush(std::knuth_b *rng) const { +#if defined(__i386__) || defined(__x86_64__) + // Note: no barriers used. + const char *p = buffer_address_ + alignment_offset_; + while (p < buffer_address_ + length_) { + if (std::uniform_int_distribution(0, 1)(*rng)) { + __builtin_ia32_clflush(p); + } + p += kCacheLineSize; + } +#endif +} + +std::string MalignBuffer::PunchedHole::ToString() const { + if (length) { + return JsonRecord("hole", Json("start", start) + ", " + + Json("length", length) + ", " + + Json("v", static_cast(v))); + } else { + return JsonNull("hole"); + } +} + +void MalignBuffer::Memset(size_t offset, unsigned char v, size_t length, + bool use_rep_stos) { + if (use_rep_stos) { + __stosb(data() + offset, v, length); + } else { + memset(data() + offset, v, length); + } +} + +void MalignBuffer::PunchHole(const PunchedHole &hole, bool use_rep_stos) { + if (hole.length) { + Memset(hole.start, hole.v, hole.length, use_rep_stos); + } +} + +// Hints to the OS to release the buffer's memory. +void MalignBuffer::MadviseDontNeed() const { + // Round up the buffer start address to a page boundary. + intptr_t start = ((intptr_t)data() + kPageSize - 1) & ~(kPageSize - 1); + // Round down the buffer end address to a page boundary. + intptr_t end = ((intptr_t)(data() + size() - 1)) & ~(kPageSize - 1); + if (end > start) { + const size_t length = end - start; + if (madvise((char *)start, length, MADV_DONTNEED) == -1) { + LOG(WARN) << "tid " + << " madvise(MADV_DONTNEED) failed: " << strerror(errno) + << " length: " << length; + } + } +} + +MalignBuffer::PunchedHole MalignBuffer::RandomPunchedHole(uint64_t seed) const { + std::knuth_b rng(seed); + MalignBuffer::PunchedHole hole; + hole.length = std::uniform_int_distribution( + 1, std::min(length_, 8192))(rng); + hole.start = + std::uniform_int_distribution(0, length_ - hole.length)(rng); + return hole; +} + +} // namespace cpu_check diff --git a/malign_buffer.h b/malign_buffer.h new file mode 100644 index 0000000..ceee77b --- /dev/null +++ b/malign_buffer.h @@ -0,0 +1,124 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_MALIGN_BUFFER_H_ +#define THIRD_PARTY_CPU_CHECK_MALIGN_BUFFER_H_ + +#include +#include + +#include "absl/strings/string_view.h" + +namespace cpu_check { + +// Data buffer supporting various alignments, copy mechanisms, and verification +// methods. +class MalignBuffer { + public: + struct PunchedHole { + std::string ToString() const; + size_t start = 0; + size_t length = 0; + unsigned char v = 0x53; + }; + + enum CopyMethod { + kMemcpy, + kRepMov, + kSseBy128, + kAvxBy256, + kAvxBy512, + }; + + // Returns name of given CopyMethod. + static std::string ToString(CopyMethod m); + + static const size_t kPageSize; + static const size_t kCacheLineSize; + + // Returns a random alignment offset in range [0..kPageSize-1]. + static size_t RandomAlignment(uint64_t seed); + + // Helper to make MSAN happy. NOP if memory sanitizer is not enabled. + static void InitializeMemoryForSanitizer(char* addr, size_t size); + + // Constructs MalignBuffer of specified capacity. + MalignBuffer(size_t capacity); + + // Constructs and initializes MalignBuffer with specified alignment and + // content. Useful for unit tests. + // REQUIRES: + // alignment_offset < kPageSize + MalignBuffer(size_t alignment_offset, absl::string_view s); + + ~MalignBuffer(); + + // Initializes buffer to specified alignment. + // REQUIRES: + // alignment_offset < kPageSize + // length <= this.capacity_. + void Initialize(size_t alignment_offset, size_t length); + + const char* data() const { return buffer_address_; } + char* data() { return buffer_address_; } + size_t size() const { return length_; } + + // REQUIRES length <= capacity_. + void resize(size_t length); + + // Compares 'this' to 'that' returning empty string if identical. + // If not identical, returns a syndrome, currently Hamming distance, + // corrupted subrange bounds, and the diffs. + std::string Syndrome(const MalignBuffer& that) const; + + // Validated data copy from source to 'this'. + // 'this' must be appropriately sized. + // Returns syndrome upon copy failure. + std::string CopyFrom(const MalignBuffer& that, CopyMethod m); + + // Unvalidated copy to 'this'. + void CopyFrom(absl::string_view src, CopyMethod m); + void CopyFrom(size_t pos, absl::string_view src, CopyMethod m); + + // Randomly flushes cache lines. + void RandomFlush(std::knuth_b* rng) const; + + // Conventional or rep;sto memset operation, according to 'use_rep_stos'. + void Memset(size_t offset, unsigned char v, size_t length, bool use_rep_stos); + + // Memsets buffer to 'hole.v', using rep;stos operation if + // 'use_rep_stos' set; + void PunchHole(const PunchedHole& hole, bool use_rep_stos); + + // Hints to the OS to release the buffer's memory. + void MadviseDontNeed() const; + + // Returns random PunchedHole within 'this'. + MalignBuffer::PunchedHole RandomPunchedHole(uint64_t seed) const; + + private: + static size_t RoundUpToPageSize(size_t k); + std::string CorruptionSyndrome(const MalignBuffer& that) const; + std::string CrackId(uint64_t) const; + + const size_t capacity_; + void* base_address_ = nullptr; + + size_t alignment_offset_ = 0; + size_t length_ = 0; // Usable length + char* buffer_address_ = nullptr; +}; + +} // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_MALIGN_BUFFER_H_ diff --git a/pattern_generator.cc b/pattern_generator.cc new file mode 100644 index 0000000..39f79f7 --- /dev/null +++ b/pattern_generator.cc @@ -0,0 +1,224 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "pattern_generator.h" + +#include + +#include +#include +#include + +#include "log.h" + +namespace cpu_check { +namespace { + +// So-called Logistic Map with parameter 4.0. +// Floating point approximation aside, if v is in the closed unit interval than +// ChaoticF1(v) is in the closed unit interval. +template +T ChaoticF1(T v) { + return 4.0 * v * (1.0 - v); +} + +// Reciprocal-like function valid over closed unit interval. +template +T Recip(T v) { + return 1.0 / (v + 0.1); +} + +// Inverse of Recip for v in closed unit interval. +template +T Unrecip(T v) { + return (1.0 / v) - 0.1; +} + +template +T ReciprocatedChaos(T v) { + return Recip(ChaoticF1(Unrecip(v))); +} + +std::vector ReadDict() { + // Dictionary search paths + static const char* dicts[] = { + "/usr/share/dict/words", + "words", + }; + std::vector words; + std::ifstream f; + + for (const auto& d : dicts) { + f.open(d, std::ifstream::in); + if (f.is_open()) break; + f.clear(); + } + + if (!f.is_open()) return words; + + LOG(DEBUG) << "Reading words."; + + std::string word; + while (!f.eof()) { + std::getline(f, word); + words.push_back(word); + } + f.close(); + LOG(DEBUG) << "Read " << words.size() << " words."; + std::sort(words.begin(), words.end(), + [](const std::string& a, const std::string& b) { + return a.size() < b.size(); + }); + return words; +} +} // namespace + +FloatingPointResults FillBufferSystematic::Generate( + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + const uint64_t pid = getpid(); + + // Format: 2 bytes of PID, 3 bytes of round number, 3 bytes of offset. + // Note: Perhaps should be AC-modulated. Perhaps should be absolute aligned + // for easier recognition. + // Note: appropriate for LE machines only. + FloatingPointResults fp; + fp.d = std::max(round, 2); + for (size_t i = 0; i * 8 < b->size(); i++) { + const size_t p = 8 * i; + const size_t k = std::min(8, b->size() - p); + const uint64_t v = + ((pid & 0xffff) << 48) | ((round & 0xffffff) << 24) | (i & 0xffffff); + for (size_t m = 0; m < k; m++) { + (b->data())[p + m] = *(reinterpret_cast(&v) + m); + } + if (exercise_floating_point) { + fp.d = ReciprocatedChaos(fp.d); + } + } + return fp; +} + +FloatingPointResults FillBufferRandom::Generate( + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + std::knuth_b rng(round); + std::uniform_int_distribution dist( + 0, std::numeric_limits::max()); + FloatingPointResults fp; + fp.f = std::max(round, 2); + size_t p = 0; + const size_t length = b->size(); + // Repeatedly append random number (one to eight) random bytes. + while (p < length) { + const size_t max_span = std::min(length - p, 8); + const size_t z = std::uniform_int_distribution(1, max_span)(rng); + const uint64_t v = dist(rng); + b->CopyFrom(p, absl::string_view(reinterpret_cast(&v), z), + copy_method); + p += z; + if (exercise_floating_point) { + fp.f = ReciprocatedChaos(fp.f); + } + } + return fp; +} + +FloatingPointResults FillBufferText::Generate( + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + std::knuth_b rng(round); + std::exponential_distribution dist(20); + FloatingPointResults fp; + fp.ld = std::max(round, 2); + const size_t bufsize = b->size(); + size_t pos = 0; + while (pos < bufsize) { + const size_t r = std::min(static_cast(dist(rng) * words_.size()), + words_.size() - 1); + const auto& word = words_[r]; + const size_t wordlen = word.size(); + if (pos + wordlen >= bufsize) { + break; + } + b->CopyFrom(pos, word, copy_method); + pos += wordlen; + if (pos < bufsize) { + b->Memset(pos, ' ', 1, use_repstos); + pos++; + } + if (exercise_floating_point) { + fp.ld = ReciprocatedChaos(fp.ld); + } + } + // Pad with spaces + b->Memset(pos, ' ', bufsize - pos, use_repstos); + return fp; +} + +FloatingPointResults FillBufferGrilledCheese::Generate( + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + std::knuth_b rng(round); + FloatingPointResults fp; + fp.f = std::max(round, 2); + fp.d = std::max(round, 2); + const size_t kAdvance = 15; + const size_t kWindow = 64; + unsigned char flavor = 0; + b->Memset(0, 0, b->size(), use_repstos); + for (int base = kWindow; base < b->size(); base += kAdvance) { + if (std::uniform_int_distribution(0, 1)(rng)) continue; + flavor++; + const size_t start = + std::uniform_int_distribution(base - kWindow, base)(rng); + const size_t end = std::uniform_int_distribution(start, base)(rng); + b->Memset(start, flavor, 1 + end - start, use_repstos); + if (exercise_floating_point) { + fp.f = ReciprocatedChaos(fp.f); + fp.d = ReciprocatedChaos(fp.d); + } + } + return fp; +} + +PatternGenerators::PatternGenerators() : words_(ReadDict()) { + if (words_.empty()) { + LOG(ERROR) << "No word list found."; + exit(1); + } + generators_.emplace_back(new FillBufferSystematic()); + generators_.emplace_back(new FillBufferRandom()); + generators_.emplace_back(new FillBufferText(words_)); + generators_.emplace_back(new FillBufferGrilledCheese()); +} + +const PatternGenerator& PatternGenerators::RandomGenerator( + uint64_t round) const { + std::knuth_b rng(round); + const size_t k = + std::uniform_int_distribution(0, generators_.size() - 1)(rng); + return *generators_[k]; +} + +FloatingPointResults PatternGenerators::Generate( + const PatternGenerator& generator, const MalignBuffer::PunchedHole& hole, + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + const FloatingPointResults f = generator.Generate( + round, copy_method, use_repstos, exercise_floating_point, b); + b->PunchHole(hole, use_repstos); + return f; +} +} // namespace cpu_check diff --git a/pattern_generator.h b/pattern_generator.h new file mode 100644 index 0000000..db8b76d --- /dev/null +++ b/pattern_generator.h @@ -0,0 +1,123 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_PATTERN_GENERATOR_H_ +#define THIRD_PARTY_CPU_CHECK_PATTERN_GENERATOR_H_ + +#include +#include + +#include "malign_buffer.h" + +namespace cpu_check { + +class PatternGenerators; + +struct FloatingPointResults { + bool operator==(const FloatingPointResults& other) const { + return f == other.f && d == other.d && ld == other.ld; + } + bool operator!=(const FloatingPointResults& other) const { + return f != other.f || d != other.d || ld != other.ld; + } + + float f = 0.0; + double d = 0.0; + long double ld = 0.0; +}; + +class PatternGenerator { + public: + virtual ~PatternGenerator() {} + virtual std::string Name() const = 0; + + virtual FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, + bool exercise_floating_point, + MalignBuffer*) const = 0; +}; + +// Fills buffer with a systematic pattern. +// Returns iterate of chaotic floating point function of 'seed', with some +// reciprocal torture. +class FillBufferSystematic : public PatternGenerator { + public: + std::string Name() const override { return "Systematic"; } + FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer*) const override; +}; + +// Fills buffer with a random pattern. +// Returns iterate of chaotic floating point function of 'seed', with some +// reciprocal torture. +class FillBufferRandom : public PatternGenerator { + public: + std::string Name() const override { return "Random"; } + FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer*) const override; +}; + +// Fills buffer with a compressible pattern. +// Returns iterate of chaotic floating point function of 'seed', with some +// reciprocal torture. +class FillBufferText : public PatternGenerator { + public: + FillBufferText(const std::vector& words) : words_(words) {} + std::string Name() const override { return "Text"; } + FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer*) const override; + + private: + const std::vector& words_; +}; + +// memset (conventional or rep;stos) randomly aligned, random width, randomly +// overlapped stretches of buffer. Constants aim to hit multiple times in +// cache lines and buffers. Untuned and based on nothing but hunches. +class FillBufferGrilledCheese : public PatternGenerator { + public: + std::string Name() const override { return "Cheese"; } + FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer*) const override; +}; + +class PatternGenerators { + public: + PatternGenerators(); + const PatternGenerator& RandomGenerator(uint64_t round) const; + + FloatingPointResults Generate(const PatternGenerator& generator, + const MalignBuffer::PunchedHole& hole, + uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer* b) const; + + const std::vector& words() const { return words_; } + + private: + const std::vector words_; + std::vector> generators_; +}; +} // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_PATTERN_GENERATOR_H_ diff --git a/silkscreen.cc b/silkscreen.cc new file mode 100644 index 0000000..d18fcc2 --- /dev/null +++ b/silkscreen.cc @@ -0,0 +1,91 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "silkscreen.h" + +#include + +#include + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "utils.h" + +namespace cpu_check { +static const size_t kPageSize = sysconf(_SC_PAGESIZE); + +Silkscreen::Silkscreen(const std::vector &tid_list) + : buffer_address_(static_cast(aligned_alloc( + kPageSize, kPageSize * ((kSize + kPageSize - 1) / kPageSize)))) { + std::knuth_b rng; + std::uniform_int_distribution dist(0, tid_list.size() - 1); + for (size_t k = 0; k < kSize; k++) { + size_t w = dist(rng); + const int o = tid_list[w]; + slot_count_[o]++; + owner_.push_back(o); + } +} + +absl::Status Silkscreen::WriteMySlots(int tid, uint64_t round) { + uint64_t j = 0; + for (size_t k = 0; k < kSize; k++) { + if (owner(k) == tid) { + *data(k) = static_cast(round); + j++; + } + } + if (j != slot_count_[tid]) { + std::string err = absl::StrCat(Json("written", j), ", ", + Json("expected", slot_count_[tid])); + return absl::Status(absl::StatusCode::kInternal, err); + } + return absl::OkStatus(); +} + +// When Silkscreen fails, it often fails, on several bad machines, +// in a surprising way: all slots owned by reading tid are +// are corrupt, in a way that suggests the previous round's +// writes never happened. Weird, and deserves some study, but +// meanwhile the log spew is suppressed by reporting only the last +// error and the error count. +absl::Status Silkscreen::CheckMySlots(int tid, uint64_t round) const { + const char expected = static_cast(round); + uint64_t slots_read = 0; + uint64_t error_count = 0; + std::string last_error; + + for (size_t k = 0; k < Silkscreen::kSize; k++) { + if (owner(k) != tid) continue; + slots_read++; + const char v = *data(k); + if (v == expected) continue; + error_count++; + last_error = absl::StrCat(Json("position", k), ", ", Json("is", v), ", ", + Json("expected", expected)); + } + if (slot_count(tid) != slots_read) { + last_error = absl::StrCat(Json("read", slots_read), ", ", + Json("expected", slot_count(tid))); + error_count++; + } + if (error_count > 0) { + return absl::Status( + absl::StatusCode::kInternal, + absl::StrCat(last_error, ", ", Json("errors", error_count))); + } else { + return absl::OkStatus(); + } +} +} // namespace cpu_check diff --git a/silkscreen.h b/silkscreen.h new file mode 100644 index 0000000..9a13590 --- /dev/null +++ b/silkscreen.h @@ -0,0 +1,63 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include "absl/status/status.h" + +#ifndef THIRD_PARTY_CPU_CHECK_SILKSCREEN_H_ +#define THIRD_PARTY_CPU_CHECK_SILKSCREEN_H_ + +namespace cpu_check { +// Rudimentary coherence/uncore tester. +// Randomly assigns each slot of a seemingly shared buffer to a single tid, +// creating only "false sharing". +// Thus each slot, regardless of alignment, must obey program order unless the +// machine is broken. +// To be toughened, e.g.: +// Widen the slots a bit +// Control the sharing more tightly, e.g. each cache line split between 2 tids +// Maybe checksum the indices to distinguish core-local compute errors from +// coherence errors, but that's perhaps easier said than done effectively. +// As it stands, it may be particularly hard to localize failures. Though that's +// always going to be a bit hard, which is the point. One technique might be +// to leave this alone and to run on subsets of cores and sockets. +class Silkscreen { + public: + static constexpr size_t kSize = 1000 * 1000; // Size of buffer + + Silkscreen(const std::vector& tid_list); + ~Silkscreen() { free(buffer_address_); } + + // Writes value derived from 'round' into all slots owned by 'tid'. + // Returns non-OK Status with JSON-formatted message upon error. + absl::Status WriteMySlots(int tid, uint64_t round); + + // Checks all slots owned by 'tid' for value appropriate to 'round'. + // Returns non-OK Status with JSON-formatted message upon error. + absl::Status CheckMySlots(int tid, uint64_t round) const; + + private: + int owner(size_t k) const { return owner_[k]; } + size_t size() const { return owner_.size(); } + int slot_count(int owner) const { return slot_count_.at(owner); } + const char* data(size_t k) const { return buffer_address_ + k; } + char* data(size_t k) { return buffer_address_ + k; } + + std::vector owner_; // const after initialization + std::map slot_count_; // const after initialization + char* const buffer_address_; +}; +} // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_SILKSCREEN_H_ diff --git a/stopper.h b/stopper.h new file mode 100644 index 0000000..8f80ec9 --- /dev/null +++ b/stopper.h @@ -0,0 +1,58 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_STOPPER_H_ +#define THIRD_PARTY_CPU_CHECK_STOPPER_H_ + +#include + +#include +#include + +#include "utils.h" + +class Stopper { + public: + // Infinite timeout if 'timeout' <= 0. + Stopper(int timeout) + : t_stop_(timeout <= 0 ? std::numeric_limits::infinity() + : TimeInSeconds() + timeout) {} + + // Returns true if time has expired or Stop has been invoked. + // Thread safe. + bool Expired() const { return stopped_ || TimeInSeconds() > t_stop_; } + + // Sleeps for the minimum of 't' and remaining run time. + // Thread safe. + void BoundedSleep(int t) const { + if (std::isinf(t_stop_)) { + sleep(t); + } else { + const double remaining = t_stop_ - TimeInSeconds(); + if (!stopped_ && remaining > 0) { + sleep(std::min(t, ceil(remaining))); + } + } + } + + // Causes timeout to expire now. + // Thread safe. + void Stop() { stopped_ = true; } + + private: + const double t_stop_; + std::atomic_bool stopped_ = false; +}; + +#endif // THIRD_PARTY_CPU_CHECK_STOPPER_H_ diff --git a/utils.cc b/utils.cc index f8e4193..69b606f 100644 --- a/utils.cc +++ b/utils.cc @@ -18,6 +18,7 @@ #include #include "log.h" +#include "absl/strings/str_cat.h" static const std::string host_name = []() { char host[256]; @@ -34,6 +35,21 @@ double TimeInSeconds() { return ((tv.tv_sec * 1e6) + tv.tv_usec) / 1e6; } +std::string HexData(const char* s, uint32_t l) { + const char d[16] = {'0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + std::string o; + o.resize(l << 1); + for (uint32_t i = 0; i < l; i++) { + uint8_t b = s[i]; + o[(i << 1)] = d[(b >> 4) & 0xf]; + o[(i << 1) + 1] = d[b & 0xf]; + } + return o; +} + +std::string HexStr(const std::string& s) { return HexData(s.data(), s.size()); } + std::string Json(const std::string& field, int v) { return "\"" + field + "\": " + std::to_string(v); } @@ -50,12 +66,12 @@ std::string JsonBool(const std::string& field, bool v) { return "\"" + field + "\": " + (v ? "true" : "false"); } -std::string Json(const std::string& field, const std::string& v) { - return "\"" + field + "\": \"" + v + "\""; +std::string Json(const std::string& field, absl::string_view v) { + return absl::StrCat("\"", field, "\": \"", v, "\""); } -std::string JsonRecord(const std::string& name, const std::string& v) { - return "\"" + name + "\": { " + v + " }"; +std::string JsonRecord(const std::string& name, absl::string_view v) { + return absl::StrCat("\"", name, "\": { ", v, " }"); } // Emits null field. diff --git a/utils.h b/utils.h index 53f8f2c..954d13e 100644 --- a/utils.h +++ b/utils.h @@ -17,15 +17,19 @@ #include +#include "absl/strings/string_view.h" double TimeInSeconds(); +std::string HexData(const char* s, uint32_t l); +std::string HexStr(const std::string& s); + std::string Json(const std::string& field, int v); std::string Json(const std::string& field, uint64_t v); std::string Json(const std::string& field, double v); std::string JsonBool(const std::string& field, bool v); -std::string Json(const std::string& field, const std::string& v); -std::string JsonRecord(const std::string& name, const std::string& v); +std::string Json(const std::string& field, absl::string_view v); +std::string JsonRecord(const std::string& name, absl::string_view v); // Emits null field. std::string JsonNull(const std::string& field);