From 83eed0a886f306610c2aae44f589789ffb9f61b9 Mon Sep 17 00:00:00 2001 From: Kevin Boyd Date: Mon, 9 Nov 2020 13:03:39 -0800 Subject: [PATCH] Sync with upstream repo. Changes include: * CPU check has been broken up into a number of small libraries * BoringSSL option has been removed * Better abseil integration --- CMakeLists.txt | 89 +- README.md | 6 +- avx.cc | 194 +++++ avx.h | 55 ++ compressor.cc | 56 ++ compressor.h | 49 ++ config.h.in | 6 - cpu_check.cc | 1940 ++++++++++-------------------------------- crypto.cc | 112 +++ crypto.h | 54 ++ hasher.cc | 95 +++ hasher.h | 96 +++ malign_buffer.cc | 382 +++++++++ malign_buffer.h | 124 +++ pattern_generator.cc | 224 +++++ pattern_generator.h | 123 +++ silkscreen.cc | 91 ++ silkscreen.h | 63 ++ stopper.h | 58 ++ utils.cc | 24 +- utils.h | 8 +- 21 files changed, 2298 insertions(+), 1551 deletions(-) create mode 100644 avx.cc create mode 100644 avx.h create mode 100644 compressor.cc create mode 100644 compressor.h create mode 100644 crypto.cc create mode 100644 crypto.h create mode 100644 hasher.cc create mode 100644 hasher.h create mode 100644 malign_buffer.cc create mode 100644 malign_buffer.h create mode 100644 pattern_generator.cc create mode 100644 pattern_generator.h create mode 100644 silkscreen.cc create mode 100644 silkscreen.h create mode 100644 stopper.h diff --git a/CMakeLists.txt b/CMakeLists.txt index ed1b6f8..e45fbef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,28 +26,6 @@ if (NOT CMAKE_BUILD_TYPE) endif(NOT CMAKE_BUILD_TYPE) -# Begin Google local change - -# We use an in tree copy of boringSSL by default. -option(USE_BORINGSSL "build with boringSSL" OFF) - -option(IN_GOOGLE3 "building in google3" OFF) - -# The vendors subdirectories may not be present. -find_path( - VENDORS_AMD_PATH - NAMES amd.cc - PATHS ${CMAKE_CURRENT_SOURCE_DIR}/vendors/amd - NO_DEFAULT_PATH -) - -find_path( - VENDORS_INTEL_PATH - NAMES intel.cc - PATHS ${CMAKE_CURRENT_SOURCE_DIR}/vendors/intel - NO_DEFAULT_PATH -) -# End Google local change # Config header configure_file ( @@ -78,24 +56,20 @@ set(CMAKE_CXX_EXTENSIONS OFF) # we want c++17 not gnu++17 add_executable(cpu_check cpu_check.cc) add_executable(crc32c_test crc32c_test.cc) +# Third party library - available as git submodule add_library(farmhash third_party/farmhash/src/farmhash.cc) + +add_library(avx avx.cc) +add_library(compressor compressor.cc) add_library(crc32c crc32c.c) +add_library(crypto crypto.cc) add_library(fvt_controller fvt_controller.cc) +add_library(hasher hasher.cc) +add_library(malign_buffer malign_buffer.cc) +add_library(pattern_generator pattern_generator.cc) +add_library(silkscreen silkscreen.cc) add_library(utils utils.cc) -# Begin Google local change -if (VENDORS_AMD_PATH) - add_library(amd ${VENDORS_AMD_PATH}/amd.cc) - add_library(hsmp ${VENDORS_AMD_PATH}/hsmp.cc) - target_link_libraries(amd hsmp pci) - set(VENDORS_LIBS ${VENDORS_LIBS} amd) -endif(VENDORS_AMD_PATH) - -if (VENDORS_INTEL_PATH) - add_library(intel ${VENDORS_INTEL_PATH}/intel.cc) - set(VENDORS_LIBS ${VENDORS_LIBS} intel) -endif(VENDORS_INTEL_PATH) -# End Google local change include(CheckCXXCompilerFlag) check_cxx_compiler_flag("-march=sandybridge" ARCH_SANDYBRIDGE) @@ -104,16 +78,20 @@ if(ARCH_SANDYBRIDGE) target_compile_options(crc32c PUBLIC -march=sandybridge) endif(ARCH_SANDYBRIDGE) -target_link_libraries(cpu_check crc32c farmhash) -# Begin Google local change -target_link_libraries(cpu_check fvt_controller ${VENDORS_LIBS} utils) -# End Google local change -target_link_libraries(crc32c_test crc32c) - if (BUILD_STATIC) set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") endif(BUILD_STATIC) +# Needs abseil +find_package(absl REQUIRED) +target_link_libraries(compressor absl::status absl::strings) +target_link_libraries(crypto absl::status absl::strings) +target_link_libraries(fvt_controller absl::strings) +target_link_libraries(malign_buffer absl::strings) +target_link_libraries(silkscreen absl::status absl::strings) +target_link_libraries(utils absl::strings) +target_link_libraries(cpu_check absl::failure_signal_handler absl::statusor absl::strings absl::symbolize) + # Needs pthreads find_package(Threads REQUIRED) target_link_libraries(cpu_check Threads::Threads) @@ -125,22 +103,17 @@ if(ZLIB_INCLUDE_DIRS) endif(ZLIB_INCLUDE_DIRS) if(ZLIB_LIBRARIES) target_link_libraries(cpu_check ${ZLIB_LIBRARIES}) + target_link_libraries(compressor ${ZLIB_LIBRARIES}) + target_link_libraries(hasher ${ZLIB_LIBRARIES}) endif(ZLIB_LIBRARIES) -# Begin Google local change -if(USE_BORINGSSL) - set(BORINGSSL_PATH ${CMAKE_CURRENT_SOURCE_DIR}/boringssl) - add_subdirectory(${BORINGSSL_PATH}) - set(BORINGSSL_INCLUDE_DIRS ${BORINGSSL_PATH}/include) - include_directories("${BORINGSSL_PATH}/include") - target_link_libraries(cpu_check ssl crypto) -else(USE_BORINGSSL) -# End Google local change # Needs OpenSSL find_package (OpenSSL REQUIRED) include_directories(${OPENSSL_INCLUDE_DIRS}) target_link_libraries(cpu_check ${OPENSSL_LIBRARIES}) +target_link_libraries(crypto ${OPENSSL_LIBRARIES}) +target_link_libraries(hasher ${OPENSSL_LIBRARIES}) # Static linking of OpenSSL may require -ldl, link it if found. find_library (dl dl) @@ -148,8 +121,18 @@ if(dl) target_link_libraries(cpu_check dl) endif(dl) -# Begin Google local change -endif(USE_BORINGSSL) -# End Google local change + + +# link malign_buffer first as it has a lot of dependencies. +target_link_libraries(malign_buffer utils) + +target_link_libraries(crc32c_test crc32c) +target_link_libraries(compressor malign_buffer) +target_link_libraries(crypto malign_buffer) +target_link_libraries(hasher crc32c farmhash malign_buffer utils) +target_link_libraries(pattern_generator malign_buffer) +target_link_libraries(silkscreen utils) + +target_link_libraries(cpu_check avx compressor crc32c crypto fvt_controller hasher malign_buffer pattern_generator silkscreen utils) install (TARGETS cpu_check DESTINATION bin) diff --git a/README.md b/README.md index fab88a5..f058d83 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,11 @@ Designed to run under Unix/Linux OS. * cmake: https://cmake.org/ * zlib -* OpenSSL/BoringSSL +* OpenSSL +* Abseil-cpp: https://github.com/abseil/abseil-cpp + +Note that Abseil must be built with the C++17 standard and include the +StatusOr package (release 2020_09_23 or later). ## Building diff --git a/avx.cc b/avx.cc new file mode 100644 index 0000000..d33f8e5 --- /dev/null +++ b/avx.cc @@ -0,0 +1,194 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "avx.h" + +#if defined(__i386__) || defined(__x86_64__) +#include +#endif + +#if defined(__i386__) || defined(__x86_64__) +#define X86_TARGET_ATTRIBUTE(s) __attribute__((target(s))) +#else +#define X86_TARGET_ATTRIBUTE(s) +#endif + +#if defined(__i386__) || defined(__x86_64__) + +bool Avx::can_do_avx() { + __builtin_cpu_init(); + return __builtin_cpu_supports("avx"); +} + +bool Avx::can_do_avx512f() { + __builtin_cpu_init(); + return __builtin_cpu_supports("avx512f"); +} + +bool Avx::can_do_fma() { + __builtin_cpu_init(); + return __builtin_cpu_supports("fma"); +} + +#else + +bool Avx::can_do_avx() { return false; } +bool Avx::can_do_avx512f() { return false; } +bool Avx::can_do_fma() { return false; } + +#endif + +std::string Avx::MaybeGoHot() { + if (std::uniform_int_distribution(0, 1)(rng_)) { + // Don't provoke. + level_ = 0; + return ""; + } + if (can_do_avx512f()) { + // Processor supports both AVX and AVX512. + level_ = std::uniform_int_distribution(0, 1)(rng_) ? 3 : 5; + } else { + // Processor supports only AVX. + level_ = 3; + } + return BurnIfAvxHeavy(); +} + +std::string Avx::BurnIfAvxHeavy() { + if (level_ == 3) { + return can_do_fma() ? Avx256FMA(kIterations) : Avx256(kIterations); + } + if (level_ == 5) { + return Avx512(kIterations); + } + return ""; +} + +// See notes for Avx512 below +X86_TARGET_ATTRIBUTE("avx") +std::string Avx::Avx256(int rounds) { +#if (defined(__i386__) || defined(__x86_64__)) + const __m256d minus_four = _mm256_set1_pd(-4.0); + __m256d x[4]; + for (int k = 0; k < 4; k++) { + x[k] = + _mm256_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); + } + double *gross_x[4] = { + reinterpret_cast(&x[0]), + reinterpret_cast(&x[1]), + reinterpret_cast(&x[2]), + reinterpret_cast(&x[3]), + }; + for (int i = 0; i < rounds; i++) { + __m256d a[4]; + a[0] = _mm256_sub_pd(_mm256_mul_pd(x[0], x[0]), x[0]); + a[1] = _mm256_sub_pd(_mm256_mul_pd(x[1], x[1]), x[1]); + a[2] = _mm256_sub_pd(_mm256_mul_pd(x[2], x[2]), x[2]); + a[3] = _mm256_sub_pd(_mm256_mul_pd(x[3], x[3]), x[3]); + x[0] = _mm256_mul_pd(minus_four, a[0]); + x[1] = _mm256_mul_pd(minus_four, a[1]); + x[2] = _mm256_mul_pd(minus_four, a[2]); + x[3] = _mm256_mul_pd(minus_four, a[3]); + } + for (int k = 1; k < 4; k++) { + for (int i = 0; i < 4; i++) { + if (gross_x[k][i] != gross_x[k][0]) { + return "avx256 pd"; + } + } + } +#endif + return ""; +} + +// See notes for Avx512 below +X86_TARGET_ATTRIBUTE("avx,fma") +std::string Avx::Avx256FMA(int rounds) { +#if (defined(__i386__) || defined(__x86_64__)) + const __m256d minus_four = _mm256_set1_pd(-4.0); + __m256d x[4]; + for (int k = 0; k < 4; k++) { + x[k] = + _mm256_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); + } + double *gross_x[4] = { + reinterpret_cast(&x[0]), + reinterpret_cast(&x[1]), + reinterpret_cast(&x[2]), + reinterpret_cast(&x[3]), + }; + for (int i = 0; i < rounds; i++) { + __m256d a[4]; + a[0] = _mm256_fmsub_pd(x[0], x[0], x[0]); + a[1] = _mm256_fmsub_pd(x[1], x[1], x[1]); + a[2] = _mm256_fmsub_pd(x[2], x[2], x[2]); + a[3] = _mm256_fmsub_pd(x[3], x[3], x[3]); + x[0] = _mm256_mul_pd(minus_four, a[0]); + x[1] = _mm256_mul_pd(minus_four, a[1]); + x[2] = _mm256_mul_pd(minus_four, a[2]); + x[3] = _mm256_mul_pd(minus_four, a[3]); + } + for (int k = 1; k < 4; k++) { + for (int i = 0; i < 4; i++) { + if (gross_x[k][i] != gross_x[k][0]) { + return "avx256 pd"; + } + } + } +#endif + return ""; +} + +// Interleave AVX512 parallel calculation of iterates of f(x) = 4x(1-x). +// Hope compiler too dumb to see through this. +X86_TARGET_ATTRIBUTE("avx512f") +std::string Avx::Avx512(int rounds) { +#if (defined(__i386__) || defined(__x86_64__)) + const __m512d minus_four = _mm512_set1_pd(-4.0); + __m512d x[4]; + for (int k = 0; k < 4; k++) { + x[k] = + _mm512_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); + } + + double *gross_x[4] = { + reinterpret_cast(&x[0]), + reinterpret_cast(&x[1]), + reinterpret_cast(&x[2]), + reinterpret_cast(&x[3]), + }; + + for (int i = 0; i < rounds; i++) { + __m512d a[4]; + a[0] = _mm512_fmsub_pd(x[0], x[0], x[0]); + a[1] = _mm512_fmsub_pd(x[1], x[1], x[1]); + a[2] = _mm512_fmsub_pd(x[2], x[2], x[2]); + a[3] = _mm512_fmsub_pd(x[3], x[3], x[3]); + x[0] = _mm512_mul_pd(minus_four, a[0]); + x[1] = _mm512_mul_pd(minus_four, a[1]); + x[2] = _mm512_mul_pd(minus_four, a[2]); + x[3] = _mm512_mul_pd(minus_four, a[3]); + } + + for (int k = 1; k < 4; k++) { + for (int i = 0; i < 7; i++) { + if (gross_x[k][i] != gross_x[k][0]) { + return "avx512 pd"; + } + } + } +#endif + return ""; +} diff --git a/avx.h b/avx.h new file mode 100644 index 0000000..7e2b473 --- /dev/null +++ b/avx.h @@ -0,0 +1,55 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#ifndef THIRD_PARTY_CPU_CHECK_AVX_H_ +#define THIRD_PARTY_CPU_CHECK_AVX_H_ + +// x86 AVX usage has complicated core power effects. This code tries +// to provoke some power transitions that don't otherwise happen. +// While it's at it, it lightly checks results, but that's not the central +// goal. ToDo: maybe toughen the correctness checking. +// +// The power policies are governed by a number of opaque parameters; this code +// is based on a lot of guesses. +// +// Not thread safe. +class Avx { + public: + static bool can_do_avx(); + static bool can_do_avx512f(); + static bool can_do_fma(); + + Avx() {} + + // Activate AVX depending on throw of the dice. + // Returns syndrome if computational error detected, empty string otherwise. + std::string MaybeGoHot(); + + // Does a bit of computing if in a "hot" mode. + // Returns syndrome if computational error detected, empty string otherwise. + std::string BurnIfAvxHeavy(); + + private: + constexpr static int kIterations = 5000; + std::string Avx256(int rounds); + std::string Avx256FMA(int rounds); + std::string Avx512(int rounds); + int level_ = 0; + std::knuth_b rng_; +}; + +#endif // THIRD_PARTY_CPU_CHECK_AVX_H_ diff --git a/compressor.cc b/compressor.cc new file mode 100644 index 0000000..4348f81 --- /dev/null +++ b/compressor.cc @@ -0,0 +1,56 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "compressor.h" + +#include "absl/status/status.h" +#include "absl/strings/str_format.h" +#include + +namespace cpu_check { + +absl::Status Zlib::Compress(const MalignBuffer &m, + MalignBuffer *compressed) const { + uLongf olen = compressBound(m.size()); + compressed->resize(olen); + int err = compress2(reinterpret_cast(compressed->data()), &olen, + reinterpret_cast(m.data()), m.size(), + Z_BEST_SPEED); + if (err != Z_OK) { + return absl::Status( + absl::StatusCode::kInternal, + absl::StrFormat("Zlib compression failed: %d srcLen: %d destLen: %d", + err, m.size(), olen)); + } + compressed->resize(olen); + return absl::OkStatus(); +} + +absl::Status Zlib::Decompress(const MalignBuffer &compressed, + MalignBuffer *m) const { + uLongf olen = m->size(); + int err = uncompress(reinterpret_cast(m->data()), &olen, + reinterpret_cast(compressed.data()), + compressed.size()); + if (err != Z_OK) { + return absl::Status( + absl::StatusCode::kInternal, + absl::StrFormat("Zlib decompression failed: %d srcLen: %d destLen: %d", + err, compressed.size(), olen)); + } + m->resize(olen); + return absl::OkStatus(); +} + +}; // namespace cpu_check diff --git a/compressor.h b/compressor.h new file mode 100644 index 0000000..b8c66b2 --- /dev/null +++ b/compressor.h @@ -0,0 +1,49 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_COMPRESSOR_H_ +#define THIRD_PARTY_CPU_CHECK_COMPRESSOR_H_ + +#include + +#include "malign_buffer.h" +#include "absl/status/status.h" + +namespace cpu_check { + +class Compressor { + public: + virtual ~Compressor() {} + virtual std::string Name() const = 0; + + // Compresses 'm' into 'compressed'. + virtual absl::Status Compress(const MalignBuffer &m, + MalignBuffer *compressed) const = 0; + + // Decompresses 'compressed' into 'm'. + virtual absl::Status Decompress(const MalignBuffer &compressed, + MalignBuffer *m) const = 0; +}; + +class Zlib : public Compressor { + public: + std::string Name() const override { return "ZLIB"; } + absl::Status Compress(const MalignBuffer &m, + MalignBuffer *compressed) const override; + absl::Status Decompress(const MalignBuffer &compressed, + MalignBuffer *m) const override; +}; + +}; // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_COMPRESSOR_H_ diff --git a/config.h.in b/config.h.in index a6641e8..e70cd9d 100644 --- a/config.h.in +++ b/config.h.in @@ -1,8 +1,2 @@ #define cpu_check_VERSION "@cpu_check_VERSION@" -// Begin Google local change -#cmakedefine IN_GOOGLE3 -#cmakedefine USE_BORINGSSL -#cmakedefine VENDORS_AMD_PATH -#cmakedefine VENDORS_INTEL_PATH -// End Google local change diff --git a/cpu_check.cc b/cpu_check.cc index 32c46e5..f250823 100644 --- a/cpu_check.cc +++ b/cpu_check.cc @@ -12,164 +12,63 @@ // See the License for the specific language governing permissions and // limitations under the License. -#undef NDEBUG -#include "config.h" - -#include -#include -#include - #include #include -#if defined(__x86_64__) || defined(__i386__) -#include -#endif #include #include #include #include +#include +#include #include -#include -#ifdef IN_GOOGLE3 -#include "third_party/zlib/zlib.h" -#else -#include -#endif #include #include -#include #include #include #include #include #include -#include #include -#include #include #include #include #include #include #include -#include #include #include #include #include -#ifdef IN_GOOGLE3 -#include "third_party/openssl/crypto.h" -#include "third_party/openssl/evp.h" -#else -#include -#include -#endif +#include "config.h" +#include "absl/debugging/failure_signal_handler.h" +#include "absl/debugging/symbolize.h" +#include "absl/status/status.h" +#include "absl/strings/str_format.h" -#ifdef IN_GOOGLE3 -#include "third_party/absl/debugging/failure_signal_handler.h" -#include "third_party/absl/debugging/symbolize.h" -#endif - -#include "crc32c.h" -#include "third_party/farmhash/src/farmhash.h" +#include "avx.h" +#include "compressor.h" +#include "crypto.h" #include "fvt_controller.h" +#include "hasher.h" #include "log.h" +#include "malign_buffer.h" +#include "pattern_generator.h" +#include "silkscreen.h" +#include "stopper.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" #include "utils.h" -#if defined(__i386__) || defined(__x86_64__) -#define X86_TARGET_ATTRIBUTE(s) __attribute__ ((target (s))) -#else -#define X86_TARGET_ATTRIBUTE(s) -#endif - #undef HAS_FEATURE_MEMORY_SANITIZER #if defined(__has_feature) -# if __has_feature(memory_sanitizer) +#if __has_feature(memory_sanitizer) #define HAS_FEATURE_MEMORY_SANITIZER -# endif +#endif #endif -// Helper to make MSAN happy. NOP if memory sanitizer is not enabled. -void InitializeMemoryForSanitizer(char* addr, size_t size) { -#ifdef HAS_FEATURE_MEMORY_SANITIZER - std::default_random_engine rnd; - std::uniform_int_distribution dist(std::numeric_limits::min(), - std::numeric_limits::max()); - for (size_t i = 0; i < size; i++) { - addr[i] = dist(rnd); - } -#endif -} - -inline void __movsb(char *dst, const char *src, size_t size) { -#if defined(__i386__) || defined(__x86_64__) - __asm__ __volatile__("rep movsb" - : "+D"(dst), "+S"(src), "+c"(size) - : - : "memory"); -#else - LOG(FATAL) << "Cannot rep;movsb"; -#endif -} - -inline void __stosb(void *dst, unsigned char c, size_t size) { -#if defined(__i386__) || defined(__x86_64__) - __asm__ __volatile__("rep stosb" - : "+D"(dst), "+c"(size) - : "a"(c) - : "memory"); -#else - LOG(FATAL) << "Cannot rep;stosb"; -#endif -} - -inline void __sse_128_memcpy(char *dst, const char *src, size_t size) { -#if (defined(__i386__) || defined(__x86_64__)) - size_t blks = size / 16; - for (int i = 0; i < blks; i++) { - _mm_storeu_si128( - reinterpret_cast<__m128i *>(dst) + i, - _mm_loadu_si128(reinterpret_cast(src) + i)); - } - memcpy(dst + blks * 16, src + blks * 16, size - blks * 16); -#else - LOG(FATAL) << "SSE not available"; -#endif -} - -X86_TARGET_ATTRIBUTE("avx") -inline void __avx_256_memcpy(char *dst, const char *src, size_t size) { -#if (defined(__i386__) || defined(__x86_64__)) - size_t blks = size / 32; - for (int i = 0; i < blks; i++) { - _mm256_storeu_si256( - reinterpret_cast<__m256i *>(dst) + i, - _mm256_loadu_si256(reinterpret_cast(src) + i)); - } - memcpy(dst + blks * 32, src + blks * 32, size - blks * 32); -#else - LOG(FATAL) << "x86 only"; -#endif -} - -X86_TARGET_ATTRIBUTE("avx512f") -inline void __avx_512_memcpy(char *dst, const char *src, size_t size) { -#if (defined(__i386__) || defined(__x86_64__)) - size_t blks = size / 64; - for (int i = 0; i < blks; i++) { - _mm512_storeu_si512( - reinterpret_cast<__m512i *>(dst) + i, - _mm512_loadu_si512(reinterpret_cast(src) + i)); - } - memcpy(dst + blks * 64, src + blks * 64, size - blks * 64); -#else - LOG(FATAL) << "x86 only"; -#endif -} - -static const double t0 = TimeInSeconds(); +using cpu_check::MalignBuffer; static void SleepForMillis(int64_t millis) { // LOG(INFO) << "sleeping " << millis; @@ -184,25 +83,6 @@ static void SleepForMillis(int64_t millis) { } } -static long pagesize = sysconf(_SC_PAGESIZE); -static long cache_line_size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); - -std::string HexData(const char *s, uint32_t l) { - const char d[16] = {'0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - std::string o; - o.resize(l << 1); - for (uint32_t i = 0; i < l; i++) { - uint8_t b = s[i]; - o[(i << 1)] = d[(b >> 4) & 0xf]; - o[(i << 1) + 1] = d[b & 0xf]; - } - return o; -} - -std::string HexStr(const std::string &s) { return HexData(s.data(), s.size()); } - -std::atomic_bool exiting(false); std::atomic_uintmax_t errorCount(0); std::atomic_uintmax_t successCount(0); static constexpr uintmax_t kErrorLimit = 2000; @@ -213,45 +93,24 @@ const bool is_x86 = true; const bool is_x86 = false; #endif -// So-called Logistic Map with parameter 4.0. -// Floating point approximation aside, if 0 < v < 1 then 0 < ChaoticF1(v) < 1. -static inline double ChaoticF1(double v) { - return 4.0 * v * (1.0 - v); -} - #if defined(__i386__) || defined(__x86_64__) -static bool can_do_avx() { - __builtin_cpu_init(); - return __builtin_cpu_supports("avx"); -} - -static bool can_do_avx512f() { - __builtin_cpu_init(); - return __builtin_cpu_supports("avx512f"); -} - -static bool can_do_fma() { - __builtin_cpu_init(); - return __builtin_cpu_supports("fma"); -} static bool can_do_fvt() { - return geteuid() == 0; // need write access to MSRs. + return geteuid() == 0; // need write access to MSRs. } #else -static bool can_do_avx() { return false; } -static bool can_do_avx512f() { return false; } -static bool can_do_fma() { return false; } + static bool can_do_fvt() { return false; } // x86-only for now. + #endif bool do_madvise = true; bool do_repmovsb = is_x86; bool do_sse_128_memcpy = is_x86; -bool do_avx_256_memcpy = can_do_avx(); -bool do_avx_512_memcpy = can_do_avx512f(); -bool do_avx_heavy = can_do_avx(); +bool do_avx_256_memcpy = Avx::can_do_avx(); +bool do_avx_512_memcpy = Avx::can_do_avx512f(); +bool do_avx_heavy = Avx::can_do_avx(); bool do_compress = true; bool do_encrypt = true; bool do_hashes = true; @@ -302,476 +161,6 @@ bool SetAffinity(int id) { return err == 0; } -std::vector ReadDict() { - // Dictionary search paths - static const char *dicts[] = { - "/usr/share/dict/words", - "words", - }; - std::vector words; - std::ifstream f; - - for (const auto &d : dicts) { - f.open(d, std::ifstream::in); - if (f.is_open()) break; - f.clear(); - } - - if (!f.is_open()) return words; - - LOG(DEBUG) << "Reading words."; - - std::string word; - while (!f.eof()) { - std::getline(f, word); - words.push_back(word); - } - f.close(); - LOG(DEBUG) << "Read " << words.size() << " words."; - std::sort(words.begin(), words.end(), - [](const std::string &a, const std::string &b) { - return a.size() < b.size(); - }); - return words; -} - -class MalignBuffer { - public: - struct PunchedHole { - std::string ToString() const; - size_t start = 0; - size_t length = 0; - unsigned char v = 0x53; - }; - - enum CopyMethod { - kMemcpy, - kRepMov, - kSseBy128, - kAvxBy256, - kAvxBy512, - }; - - static std::string ToString(CopyMethod m) { - switch (m) { - case kMemcpy: - return "memcpy"; - case kRepMov: - return "rep;mov"; - case kSseBy128: - return "sse:128"; - case kAvxBy256: - return "avx:256"; - case kAvxBy512: - return "avx:512"; - } - } - - // Provides buffer with specified alignment. - MalignBuffer(size_t capacity) - : capacity_(capacity), - base_address_(aligned_alloc(pagesize, capacity + pagesize)) { - assert(base_address_); - // There are lots of places that use unitialized MalignBuffer. So just - // fill some pseudo-random bytes if cpu_check is compiled with msan. - InitializeMemoryForSanitizer(static_cast(base_address_), capacity_); - } - ~MalignBuffer() { free(base_address_); } - - // REQUIRES: alignment_offset + length <= capacity_. - void Initialize(size_t alignment_offset, size_t length); - - const char* data() const { return buffer_address_; } - char* data() { return buffer_address_; } - size_t size() const { return length_; } - - // REQUIRES alignment_offset + length <= capacity_. - void resize(size_t length); - - // Compares 'this' to 'that' returning empty string if identical. - // If not identical, returns a syndrome, currently Hamming distance, - // corrupted subrange bounds, and the diffs. - std::string Syndrome(const MalignBuffer& that) const; - - // Validated data copy from source to 'this'. - // 'this' must be appropriately sized. - // Returns syndrome upon copy failure. - std::string CopyFrom(const MalignBuffer& that, CopyMethod m); - - // Unvalidated copy to 'this'. - void CopyFrom(const char* src, size_t length, CopyMethod m); - void CopyFrom(size_t pos, const char* src, size_t length, CopyMethod m); - - // Randomly flushes cache lines. - void RandomFlush(std::knuth_b* rng) const; - - // Conventional or rep;sto memset operation, according to 'use_rep_stos'. - void Memset(size_t offset, unsigned char v, size_t length, bool use_rep_stos); - - // Memsets buffer to 'hole.v', using rep;stos operation if - // 'use_rep_stos' set; - void PunchHole(const PunchedHole& hole, bool use_rep_stos); - - private: - std::string CorruptionSyndrome(const MalignBuffer& that) const; - std::string CrackId(uint64_t) const; - - const size_t capacity_; - void* base_address_ = nullptr; - - size_t alignment_offset_ = 0; - size_t length_ = 0; // Usable length - char* buffer_address_ = nullptr; -}; - -void MalignBuffer::Initialize(size_t alignment_offset, size_t length) { - assert(alignment_offset + length <= capacity_); - alignment_offset_ = alignment_offset; - length_ = length; - buffer_address_ = static_cast(base_address_) + alignment_offset_; -} - -void MalignBuffer::resize(size_t length) { - Initialize(alignment_offset_, length); -} - -std::string MalignBuffer::CopyFrom(const MalignBuffer& that, CopyMethod m) { - CopyFrom(that.data(), that.size(), m); - return Syndrome(that); -} - -void MalignBuffer::CopyFrom(const char* src, size_t length, CopyMethod m) { - assert(size() == length); - CopyFrom(0, src, length, m); -} - -void MalignBuffer::CopyFrom(size_t pos, const char* src, size_t length, - CopyMethod m) { -assert(pos + length <= size()); -switch (m) { - case kMemcpy: - // Assumes memcpy doesn't use rep;movsb; false in lots of environments. - memcpy(data() + pos, src, length); - break; - case kRepMov: - __movsb(data() + pos, src, length); - break; - case kSseBy128: - __sse_128_memcpy(data() + pos, src, length); - break; - case kAvxBy256: - __avx_256_memcpy(data() + pos, src, length); - break; - case kAvxBy512: - __avx_512_memcpy(data() + pos, src, length); - break; -} -} - -std::string MalignBuffer::Syndrome(const MalignBuffer& that) const { - std::stringstream s; - std::string syndrome = CorruptionSyndrome(that); - if (syndrome.empty()) return ""; - s << syndrome << ", \"this\": \"" << static_cast(data()) - << "\", " - << "\"that\": \"" << static_cast(that.data()) << "\""; - return s.str(); -} - -std::string MalignBuffer::CorruptionSyndrome(const MalignBuffer& that) const { - std::stringstream s; - if (size() != that.size()) { - s << Json("unequalSizeThis", size()) << ", " - << Json("unequalSizeThat", that.size()); - return s.str(); - } - bool failed_memcmp = memcmp(data(), that.data(), that.size()); - - int wrong_bytes = 0; - int wrong_bits = 0; - int byte_faults = 0; - int first_wrong = INT_MAX; - int last_wrong = INT_MIN; - std::vector lane_errors(8, 0); - for (size_t i = 0; i < size(); i++) { - unsigned char a = *(data() + i); - unsigned char b = *(that.data() + i); - unsigned char d = a ^ b; - if (d) { - first_wrong = std::min(first_wrong, i); - last_wrong = std::max(last_wrong, i); - byte_faults |= d; - wrong_bytes++; - wrong_bits += __builtin_popcount(d); - for (size_t i = 0; i < 8; i++) { - if ((d >> i) & 1) { - lane_errors[i]++; - } - } - } - } - if (wrong_bits || wrong_bytes) { - const int range_width = (last_wrong - first_wrong) + 1; - s << Json("cmpResult", - (failed_memcmp ? "Failed_Memcmp" : "**Passed_Memcmp**")) - << ", " << Json("wrongByteCount", wrong_bytes) << ", " - << Json("wrongBitCount", wrong_bits) << ", " - << Json("corruptionWidth", range_width) << ", " - << Json("corruptStart", first_wrong) << ", " - << Json("corruptByteBitMask", byte_faults) << ", " - << "\"byBitLane\": ["; - for (size_t i = 0; i < 8; i++) { - if (i) s << ", "; - s << lane_errors[i]; - } - s << " ] "; - // Dump up to 64 corrupted locations. - std::stringstream dump; - dump << " \"byteErrors\": [ " << std::hex; - uint64_t buf_a = 0; - uint64_t buf_b = 0; - for (size_t k = 0; k < std::min(64, range_width); k++) { - uint8_t a = *(data() + first_wrong + k); - uint8_t b = *(that.data() + first_wrong + k); - if (k) dump << ", "; - dump << "[ " << std::setw(2) << "\"0x" << static_cast(a) << "\", " - << std::setw(2) << "\"0x" << static_cast(b) << "\" "; - buf_a = (buf_a >> 8) | static_cast(a) << 56; - buf_b = (buf_b >> 8) | static_cast(b) << 56; - if ((k >= 7) && (7 == ((first_wrong + k) % 8))) { - dump << ", " << CrackId(buf_a) << ", " << CrackId(buf_b); - buf_a = 0; - buf_b = 0; - } - dump << " ]"; - } - dump << " ] "; - return s.str() + ", " + dump.str(); - } else { - if (!failed_memcmp) return ""; - return Json("cmpResult", "**Failed_Memcmp**"); - } -} - -std::string MalignBuffer::CrackId(uint64_t v) const { - std::stringstream s; - s << std::hex << " [\"0x" << std::setw(4) << (v >> 48) << "\", \"0x" - << std::setw(6) << ((v >> 24) & 0xffffff) << "\", \"0x" << std::setw(6) - << (v & 0xffffff) << "\"]"; - return s.str(); -} - -void MalignBuffer::RandomFlush(std::knuth_b* rng) const { -#if defined(__i386__) || defined(__x86_64__) - // Note: no barriers used. - const char* p = buffer_address_ + alignment_offset_; - while (p < buffer_address_ + length_) { - if (std::uniform_int_distribution(0, 1)(*rng)) { - __builtin_ia32_clflush(p); - } - p += cache_line_size; - } -#endif -} - -std::string MalignBuffer::PunchedHole::ToString() const { - if (length) { - return JsonRecord("hole", - Json("start", start) + ", " + Json("length", length) - + ", " + Json("v", static_cast(v))); - } else { - return JsonNull("hole"); - } -} - -void MalignBuffer::Memset(size_t offset, unsigned char v, size_t length, - bool use_rep_stos) { - if (use_rep_stos) { - __stosb(data() + offset, v, length); - } else { - memset(data() + offset, v, length); - } -} - -void MalignBuffer::PunchHole(const PunchedHole& hole, bool use_rep_stos) { - if (hole.length) { - Memset(hole.start, hole.v, hole.length, use_rep_stos); - } -} - -// x86 AVX usage has complicated core power effects. This code tries -// to provoke some power transitions that don't otherwise happen. -// While it's at it, it lightly checks results, but that's not the central -// goal. ToDo: maybe toughen the correctness checking. -// -// The power policies are governed by a number of opaque parameters; this code -// is based on a lot of guesses. - -class Avx { - public: - Avx() {} - - // Activate AVX depending on throw of the dice. - // Returns syndrome if computational error detected. - std::string MaybeGoHot(); - - // Does a bit of computing if in a "hot" mode. - // Returns syndrome if computational error detected. - std::string BurnIfAvxHeavy(); - - private: - constexpr static int kIterations = 5000; - std::string Avx256(int rounds); - std::string Avx256FMA(int rounds); - std::string Avx512(int rounds); - int level_ = 0; - std::knuth_b rng_; -}; - -std::string Avx::MaybeGoHot() { - if (std::uniform_int_distribution(0, 1)(rng_)) { - // Don't provoke. - level_ = 0; - return ""; - } - if (can_do_avx512f()) { - // Processor supports both AVX and AVX512. - level_ = std::uniform_int_distribution(0, 1)(rng_) ? 3 : 5; - } else { - // Processor supports only AVX. - level_ = 3; - } - return BurnIfAvxHeavy(); -} - -std::string Avx::BurnIfAvxHeavy() { - if (level_ == 3) { - return can_do_fma() ? Avx256FMA(kIterations) : Avx256(kIterations); - } - if (level_ == 5) { - return Avx512(kIterations); - } - return ""; -} - -// See notes for Avx512 below -X86_TARGET_ATTRIBUTE("avx") -std::string Avx::Avx256(int rounds) { -#if (defined(__i386__) || defined(__x86_64__)) - const __m256d minus_four = _mm256_set1_pd(-4.0); - __m256d x[4]; - for (int k = 0; k < 4; k++) { - x[k] = - _mm256_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); - } - double *gross_x[4] = { - reinterpret_cast(&x[0]), - reinterpret_cast(&x[1]), - reinterpret_cast(&x[2]), - reinterpret_cast(&x[3]), - }; - for (int i = 0; i < rounds; i++) { - __m256d a[4]; - a[0] = _mm256_sub_pd(_mm256_mul_pd(x[0], x[0]), x[0]); - a[1] = _mm256_sub_pd(_mm256_mul_pd(x[1], x[1]), x[1]); - a[2] = _mm256_sub_pd(_mm256_mul_pd(x[2], x[2]), x[2]); - a[3] = _mm256_sub_pd(_mm256_mul_pd(x[3], x[3]), x[3]); - x[0] = _mm256_mul_pd(minus_four, a[0]); - x[1] = _mm256_mul_pd(minus_four, a[1]); - x[2] = _mm256_mul_pd(minus_four, a[2]); - x[3] = _mm256_mul_pd(minus_four, a[3]); - } - for (int k = 1; k < 4; k++) { - for (int i = 0; i < 4; i++) { - if (gross_x[k][i] != gross_x[k][0]) { - return "avx256 pd"; - } - } - } -#endif - return ""; -} - -// See notes for Avx512 below -X86_TARGET_ATTRIBUTE("avx,fma") -std::string Avx::Avx256FMA(int rounds) { -#if (defined(__i386__) || defined(__x86_64__)) - const __m256d minus_four = _mm256_set1_pd(-4.0); - __m256d x[4]; - for (int k = 0; k < 4; k++) { - x[k] = - _mm256_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); - } - double *gross_x[4] = { - reinterpret_cast(&x[0]), - reinterpret_cast(&x[1]), - reinterpret_cast(&x[2]), - reinterpret_cast(&x[3]), - }; - for (int i = 0; i < rounds; i++) { - __m256d a[4]; - a[0] = _mm256_fmsub_pd(x[0], x[0], x[0]); - a[1] = _mm256_fmsub_pd(x[1], x[1], x[1]); - a[2] = _mm256_fmsub_pd(x[2], x[2], x[2]); - a[3] = _mm256_fmsub_pd(x[3], x[3], x[3]); - x[0] = _mm256_mul_pd(minus_four, a[0]); - x[1] = _mm256_mul_pd(minus_four, a[1]); - x[2] = _mm256_mul_pd(minus_four, a[2]); - x[3] = _mm256_mul_pd(minus_four, a[3]); - } - for (int k = 1; k < 4; k++) { - for (int i = 0; i < 4; i++) { - if (gross_x[k][i] != gross_x[k][0]) { - return "avx256 pd"; - } - } - } -#endif - return ""; -} - -// Interleave AVX512 parallel calculation of iterates of f(x) = 4x(1-x). -// Hope compiler too dumb to see through this. -X86_TARGET_ATTRIBUTE("avx512f") -std::string Avx::Avx512(int rounds) { -#if (defined(__i386__) || defined(__x86_64__)) - const __m512d minus_four = _mm512_set1_pd(-4.0); - __m512d x[4]; - for (int k = 0; k < 4; k++) { - x[k] = - _mm512_set1_pd(std::uniform_real_distribution(0.0, 1.0)(rng_)); - } - - double *gross_x[4] = { - reinterpret_cast(&x[0]), - reinterpret_cast(&x[1]), - reinterpret_cast(&x[2]), - reinterpret_cast(&x[3]), - }; - - for (int i = 0; i < rounds; i++) { - __m512d a[4]; - a[0] = _mm512_fmsub_pd(x[0], x[0], x[0]); - a[1] = _mm512_fmsub_pd(x[1], x[1], x[1]); - a[2] = _mm512_fmsub_pd(x[2], x[2], x[2]); - a[3] = _mm512_fmsub_pd(x[3], x[3], x[3]); - x[0] = _mm512_mul_pd(minus_four, a[0]); - x[1] = _mm512_mul_pd(minus_four, a[1]); - x[2] = _mm512_mul_pd(minus_four, a[2]); - x[3] = _mm512_mul_pd(minus_four, a[3]); - } - - for (int k = 1; k < 4; k++) { - for (int i = 0; i < 7; i++) { - if (gross_x[k][i] != gross_x[k][0]) { - return "avx512 pd"; - } - } - } -#endif - return ""; -} - // Produces noise of all kinds by running intermittently. // There's a coarse cycle with four fine phases: // Phase 0: Off @@ -822,122 +211,41 @@ class NoiseScheduler { } }; -// Rudimentary coherence/uncore tester. -// Randomly assigns each slot of a seemingly shared buffer to a single tid, -// creating only "false sharing". -// Thus each slot, regardless of alignment, must obey program order unless the -// machine is broken. -// To be toughened, e.g.: -// Widen the slots a bit -// Control the sharing more tightly, e.g. each cache line split between 2 tids -// Maybe checksum the indices to distinguish core-local compute errors from -// coherence errors, but that's perhaps easier said than done effectively. -// As it stands, it may be particularly hard to localize failures. Though that's -// always going to be a bit hard, which is the point. One technique might be -// to leave this alone and to run on subsets of cores and sockets. -class Silkscreen { - public: - static constexpr size_t kSize = 1000 * 1000; // Size of buffer - - Silkscreen(const std::vector &tid_list); - ~Silkscreen() { free(buffer_address_); } - - // Writes value derived from 'round' into all slots owned by 'tid'. - // Returns number of slots written. - uint64_t WriteMySlots(int tid, uint64_t round); - - // Returns JSON-formatted error string if slot 'k' belongs to 'tid' and has - // value not properly corresponding with 'round'. - // Returns "=" if slot 'k' belongs to 'tid' and has expected value. - // Otherwise, if slot 'k' does not belong to 'tid', returns empty string. - std::string CheckMySlot(int tid, uint64_t round, size_t k) const; - - private: - int owner(size_t k) const { return owner_[k]; } - size_t size() const { return owner_.size(); } - const char* data(size_t k) const { return buffer_address_ + k; } - char* data(size_t k) { return buffer_address_ + k; } - - std::vector owner_; // const after initialization - char* buffer_address_ = nullptr; -}; - -Silkscreen::Silkscreen(const std::vector &tid_list) - : buffer_address_(static_cast(aligned_alloc( - pagesize, - pagesize * ((kSize + pagesize - 1) / pagesize) ))) { - std::knuth_b rng; - std::uniform_int_distribution dist(0, tid_list.size() - 1); - for (size_t k = 0; k < kSize; k++) { - size_t w = dist(rng); - owner_.push_back(tid_list[w]); - } -} - -uint64_t Silkscreen::WriteMySlots(int tid, uint64_t round) { - uint64_t j = 0; - for (size_t k = 0; k < kSize; k++) { - if (owner(k) == tid) { - *data(k) = static_cast(round); - j++; - } - } - return j; -} - -std::string Silkscreen::CheckMySlot(int tid, uint64_t round, size_t k) const { - if (owner(k) != tid) return ""; - const int v = *data(k); - const int w = static_cast(round); - if (v == w) return "="; - return - Json("position", k) + ", " + Json("is", v) + ", " + Json("expected", w); -} - class Worker { public: - // Does not take ownership of 'silkscreen'. - Worker(int pid, const std::vector *words, - std::vector tid_list, int tid, Silkscreen *silkscreen) - : pid_(pid), tid_(tid), words_(words), tid_list_(tid_list), - silkscreen_(silkscreen), rndeng_(std::random_device()()) { - } + // Does not take ownership of 'silkscreen' or 'stopper'. + Worker(int pid, std::vector tid_list, int tid, + cpu_check::Silkscreen *silkscreen, Stopper *stopper) + : pid_(pid), + tid_(tid), + tid_list_(tid_list), + silkscreen_(silkscreen), + stopper_(stopper), + rndeng_(std::random_device()()) {} ~Worker() {} void Run(); private: static constexpr size_t kBufMin = 12; #ifdef HAVE_FEATURE_MEMORY_SANITIZER - // Use smaller buffers if cpu_check is built with msan. Otherwise - // we will time out in testing. - static constexpr size_t kBufMax = 1 << 16; // 64 KiB + // Use smaller buffers if cpu_check is built with msan. Otherwise + // we will time out in testing. + static constexpr size_t kBufMax = 1 << 16; // 64 KiB #else static constexpr size_t kBufMax = 1 << 20; // 1 MiB #endif - struct FloatingPointResults { - bool operator!=(const FloatingPointResults& other) const { - return d != other.d; - } - - double d = 0.0; - }; - - typedef struct { - const char *name; - FloatingPointResults (Worker::*func)( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer*) const; - } generatorItem; - struct BufferSet { void Alloc(std::unique_ptr *p) { - const size_t kBufCap = kBufMax + 23 * pagesize; if (!*p) { - p->reset(new MalignBuffer(kBufCap)); + // Allocate buffer larger than kBufMax because compression can, in some + // cases of random plain text, cause some expansion. + p->reset(new MalignBuffer(2 * kBufMax + 1024)); } } + // The buffers holding successive data transformations. Some transformations + // are optional, so some buffers may be unused. std::unique_ptr original; std::unique_ptr compressed; std::unique_ptr encrypted; @@ -945,73 +253,103 @@ class Worker { std::unique_ptr decrypted; std::unique_ptr decompressed; std::unique_ptr re_made; + + // The plain-text buffer that was encrypted, if encryption was performed. + MalignBuffer *pre_encrypted = nullptr; + // The result of the series of transformations up to, but not including, + // the final copy. + MalignBuffer *pre_copied = nullptr; }; - FloatingPointResults FillBufferSystematic( - uint32_t unused_seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; - FloatingPointResults FillBufferRandomData( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; - FloatingPointResults FillBufferRandomText( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; - FloatingPointResults FillBufferGrilledCheese( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; + // Computation parameters. + struct Choices { + bool madvise; + bool use_repstos; + bool exercise_floating_point; + MalignBuffer::CopyMethod copy_method; + cpu_check::PatternGenerator const *pattern_generator = nullptr; + cpu_check::Hasher const *hasher = nullptr; + size_t buf_size; + MalignBuffer::PunchedHole hole; + std::string summary; + }; - void MadviseDontNeed(const MalignBuffer &s) const; + // Various checksums produced in the course of the series of data + // transformations. + struct Checksums { + std::string hash_value; + cpu_check::FloatingPointResults floating_point_results; + cpu_check::Crypto::CryptoPurse crypto_purse; + }; + + uint64_t Seed() { return std::uniform_int_distribution()(rndeng_); } size_t Alignment(); void MaybeFlush(const MalignBuffer &s); + // Attempts to schedules CPU frequency using the Worker's. // FVTController object. Returns the scheduled frequency or // 0 if there is no FVTController available. int ScheduledMHz() const; MalignBuffer::CopyMethod CopyMethod(); std::string FVT() const; - MalignBuffer::PunchedHole PunchedHole(size_t bufsize); - FloatingPointResults GenerateData( - const generatorItem& generator, - const MalignBuffer::PunchedHole& hole, - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const; + + // Returns 'Choices' that seed the data transformations. + Choices MakeChoices(BufferSet *b); + + // Performs a series of data transformations. + // Returns an error status if computation is detected to be corrupt. + absl::StatusOr DoComputations(const std::string &writer_ident, + const Choices &choices, + BufferSet *b); + + // Inverts DoComputations, checking correctness of results. + // Returns an error status if corruption is detected. + absl::Status CheckComputations(const std::string &writer_reader_ident, + const Choices &choices, + const Checksums &checksums, BufferSet *b); // Emits a failure record. // TODO: bump error count here, and maybe log, instead of at every // call site. - std::string Jfail(const std::string &err, const std::string &v) { + std::string Jfail(absl::string_view err, const absl::string_view v) { if (errorCount > error_limit) { - exiting = true; + stopper_->Stop(); LOG(INFO) << "I am quitting after " << errorCount << " errors"; } - return "{ " + JsonRecord("fail", Json("err", err) + ", " + v) + ", " + - JTag() + " }"; + return "{ " + JsonRecord("fail", absl::StrCat(Json("err", err), ", ", v)) + + ", " + JTag() + " }"; } - // Array of random data generators. - static const std::vector kGenerators; + absl::Status ReturnError(absl::string_view err, const absl::string_view v) { + return absl::Status(absl::StatusCode::kInternal, Jfail(err, v)); + } + + std::string Suspect(int tid) { + return absl::StrFormat("Suspect LPU: %d", tid); + } + + // Returns two tids to be used for checking computation. If 'do_hop', + // CheckerTids avoids duplication and avoids 'tid_' if it can. + std::vector CheckerTids(); const uint64_t pid_; const int tid_; - const std::vector *words_; const std::vector tid_list_; - Silkscreen* const silkscreen_; + cpu_check::Silkscreen *const silkscreen_; + Stopper *const stopper_; // We don't really need "good" random numbers. // std::mt19937_64 rndeng_; std::knuth_b rndeng_; uint64_t round_ = 0; + Avx avx_; + cpu_check::PatternGenerators pattern_generators_; + cpu_check::Hashers hashers_; + cpu_check::Zlib zlib_; std::unique_ptr fvt_controller_; }; -const std::vector Worker::kGenerators = { - {"SYSTEMATIC", &Worker::FillBufferSystematic}, - {"DATA", &Worker::FillBufferRandomData}, - {"TEXT", &Worker::FillBufferRandomText}, - {"CHEESE", &Worker::FillBufferGrilledCheese}, -}; - std::string Worker::FVT() const { if (fvt_controller_ == nullptr) return ""; return fvt_controller_->FVT(); @@ -1026,8 +364,8 @@ int Worker::ScheduledMHz() const { // User-specified fixed frequency. return fixed_min_frequency; } - if (!do_freq_sweep && !do_freq_hi_lo - && !fixed_min_frequency && !fixed_max_frequency) { + if (!do_freq_sweep && !do_freq_hi_lo && !fixed_min_frequency && + !fixed_max_frequency) { // Run at maximum frequency. return fvt_controller_->limit_mHz(); } @@ -1053,128 +391,7 @@ int Worker::ScheduledMHz() const { } } -// Fills 'b' with a systematic pattern. -// Returns iterate of chaotic floating point function of 'seed', with some -// reciprocal torture. -Worker::FloatingPointResults Worker::FillBufferSystematic( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const { - // Format: 2 bytes of PID, 3 bytes of round number, 3 bytes of offset. - // Note: Perhaps should be AC-modulated. Perhaps should be absolute aligned - // for easier recognition. - // Note: appropriate for LE machines only. - FloatingPointResults fp; - fp.d = std::max(seed, 2); - for (size_t i = 0; i * 8 < b->size(); i++) { - const size_t p = 8 * i; - const size_t k = std::min(8, b->size() - p); - const uint64_t v = - ((pid_ & 0xffff) << 48) - | ((round_ & 0xffffff) << 24) | (i & 0xffffff); - for (size_t m = 0; m < k; m++) { - (b->data())[p + m] = *(reinterpret_cast(&v) + m); - } - fp.d = 1.0 / ChaoticF1(1.0 / fp.d); - } - fp.d = 1.0 / fp.d; - return fp; -} - -Worker::FloatingPointResults Worker::FillBufferRandomData( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const { - std::knuth_b rng(seed); - std::uniform_int_distribution - dist(0, std::numeric_limits::max()); - size_t p = 0; - const size_t length = b->size(); - // Repeatedly append random number (one to eight) random bytes. - while (p < length) { - const size_t max_span = std::min(length - p, 8); - const size_t z = std::uniform_int_distribution(1, max_span)(rng); - const uint64_t v = dist(rng); - b->CopyFrom(p, reinterpret_cast(&v), z, copy_method); - p += z; - } - return FloatingPointResults(); -} - -Worker::FloatingPointResults Worker::FillBufferRandomText( - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, MalignBuffer* b) const { - std::knuth_b rng(seed); - std::exponential_distribution dist(20); - const size_t bufsize = b->size(); - size_t pos = 0; - while (pos < bufsize) { - const size_t r = std::min(static_cast(dist(rng) * words_->size()), - words_->size() - 1); - const auto &word = (*words_)[r]; - const size_t wordlen = word.size(); - if (pos + wordlen >= bufsize) { - break; - } - b->CopyFrom(pos, word.c_str(), wordlen, copy_method); - pos += wordlen; - if (pos < bufsize) { - b->Memset(pos, ' ', 1, use_repstos); - pos++; - } - } - // Pad with spaces - b->Memset(pos, ' ', bufsize - pos, use_repstos); - return FloatingPointResults(); -} - -// memset (conventional or rep;stos) randomly aligned, random width, randomly -// overlapped stretches of buffer. Constants aim to hit multiple times in cache -// lines and buffers. Untuned and based on nothing but hunches. -Worker::FloatingPointResults Worker::FillBufferGrilledCheese( - uint32_t seed, MalignBuffer::CopyMethod copy_method, bool use_repstos, - MalignBuffer* b) const { - std::knuth_b rng(seed); - const size_t kAdvance = 15; - const size_t kWindow = 64; - unsigned char flavor = 0; - b->Memset(0, 0, b->size(), use_repstos); - for (int base = kWindow; base < b->size(); base += kAdvance) { - if (std::uniform_int_distribution(0, 1)(rng)) continue; - flavor++; - const size_t start = - std::uniform_int_distribution(base - kWindow, base)(rng); - const size_t end = std::uniform_int_distribution(start, base)(rng); - b->Memset(start, flavor, 1 + end - start, use_repstos); - } - return FloatingPointResults(); -} - -Worker::FloatingPointResults Worker::GenerateData( - const generatorItem& generator, - const MalignBuffer::PunchedHole& hole, - uint32_t seed, MalignBuffer::CopyMethod copy_method, - bool use_repstos, - MalignBuffer* b) const { - const FloatingPointResults f = - (this->*generator.func)(seed, copy_method, use_repstos, b); - b->PunchHole(hole, use_repstos); - return f; -} - -// Hints to the OS to release the buffer's memory. -void Worker::MadviseDontNeed(const MalignBuffer &s) const { - // Round up the buffer start address to a page boundary. - intptr_t start = ((intptr_t) s.data() + pagesize - 1) & ~(pagesize - 1); - // Round down the buffer end address to a page boundary. - intptr_t end = ((intptr_t) (s.data() + s.size() - 1)) & ~(pagesize - 1); - if (end - start >= pagesize) { - if (madvise((char *)start, end - start, MADV_DONTNEED) == -1) { - LOG(WARN) << "tid " << tid_ - << " madvise(MADV_DONTNEED) failed: " << strerror(errno); - } - } -} - -void Worker::MaybeFlush(const MalignBuffer& s) { +void Worker::MaybeFlush(const MalignBuffer &s) { // Half the time, tell the OS to release the destination buffer. if (do_flush && std::uniform_int_distribution(0, 1)(rndeng_)) { s.RandomFlush(&rndeng_); @@ -1182,8 +399,7 @@ void Worker::MaybeFlush(const MalignBuffer& s) { } size_t Worker::Alignment() { - return do_misalign ? - std::uniform_int_distribution(0, pagesize)(rndeng_) : 0; + return do_misalign ? MalignBuffer::RandomAlignment(Seed()) : 0; } MalignBuffer::CopyMethod Worker::CopyMethod() { @@ -1202,145 +418,252 @@ MalignBuffer::CopyMethod Worker::CopyMethod() { return v[k]; } -MalignBuffer::PunchedHole Worker::PunchedHole(size_t bufsize) { - MalignBuffer::PunchedHole hole; - hole.length = - std::uniform_int_distribution( - 1, std::min(bufsize, 8192))(rndeng_); - hole.start = - std::uniform_int_distribution( - 0, bufsize - hole.length)(rndeng_); - return hole; +Worker::Choices Worker::MakeChoices(BufferSet *b) { + Choices c; + c.madvise = do_madvise && std::uniform_int_distribution(0, 1)(rndeng_); + c.copy_method = CopyMethod(); + + c.use_repstos = + do_repstosb && std::uniform_int_distribution(0, 1)(rndeng_); + + // Exercise floating point (in pattern generators) relatively rarely because + // it's expensive and it doesn't catch a lot of machines. + c.exercise_floating_point = + std::uniform_int_distribution(0, 20)(rndeng_) == 0; + + c.pattern_generator = &pattern_generators_.RandomGenerator(round_); + c.hasher = &hashers_.RandomHasher(round_); + + c.buf_size = std::uniform_int_distribution(kBufMin, kBufMax)(rndeng_); + if (!b->original) b->Alloc(&b->original); + b->original->Initialize(Alignment(), c.buf_size); + c.hole = b->original->RandomPunchedHole(Seed()); + + c.summary = absl::StrCat( + Json("pattern", c.pattern_generator->Name()), ", ", + Json("hash", c.hasher->Name()), ", ", + Json("copy", MalignBuffer::ToString(c.copy_method)), ", ", + Json("memset", c.use_repstos ? "rep;sto" : "memset"), ", ", + JsonBool("madvise", c.madvise), ", ", Json("size", c.buf_size), ", ", + Json("pid", pid_), ", ", Json("round", round_), ", ", c.hole.ToString()); + + return c; } -std::string OpenSSL_Hash(const MalignBuffer &s, const EVP_MD *type) { - EVP_MD_CTX *ctx; - ctx = EVP_MD_CTX_create(); - EVP_DigestInit_ex(ctx, type, nullptr); - std::string hash; - hash.resize(EVP_MD_CTX_size(ctx)); - InitializeMemoryForSanitizer(hash.data(), EVP_MD_CTX_size(ctx)); - EVP_DigestUpdate(ctx, s.data(), s.size()); - EVP_DigestFinal_ex(ctx, (uint8_t *)&hash[0], nullptr); - EVP_MD_CTX_destroy(ctx); - return HexStr(hash); +absl::StatusOr Worker::DoComputations( + const std::string &writer_ident, const Choices &choices, BufferSet *b) { + Checksums checksums; + if (do_avx_heavy) { + const std::string e = avx_.MaybeGoHot(); + if (!e.empty()) { + return ReturnError(e, writer_ident); + } + } + + if (do_ssl_self_check) { + auto s = cpu_check::Crypto::SelfTest(); + if (!s.ok()) { + return ReturnError(s.message(), writer_ident); + } + } + + auto s = silkscreen_->WriteMySlots(tid_, round_); + if (!s.ok()) { + return ReturnError("Silkscreen", + absl::StrCat(s.message(), ", ", writer_ident)); + } + + if (do_avx_heavy) { + // If we tried to do AVX heavy stuff. Try to run AVX heavy again to try + // to spike current. + const std::string e = avx_.BurnIfAvxHeavy(); + if (!e.empty()) { + return ReturnError(e, writer_ident); + } + } + + checksums.floating_point_results = pattern_generators_.Generate( + *choices.pattern_generator, choices.hole, round_, choices.copy_method, + choices.use_repstos, choices.exercise_floating_point, b->original.get()); + MaybeFlush(*b->original); + + MalignBuffer *head = b->original.get(); + + if (do_hashes) { + checksums.hash_value = choices.hasher->Hash(*head); + } + + if (do_compress) { + // Run our randomly chosen compressor. + if (!b->compressed) b->Alloc(&b->compressed); + b->compressed->Initialize(Alignment(), choices.buf_size); + MaybeFlush(*b->compressed); + + const auto s = zlib_.Compress(*head, b->compressed.get()); + if (!s.ok()) { + return ReturnError( + "Compression", + absl::StrCat(Json("syndrome", s.message()), ", ", writer_ident)); + } + MaybeFlush(*b->compressed); + head = b->compressed.get(); + } + + b->pre_encrypted = head; + if (do_encrypt) { + // Encrypt. + if (!b->encrypted) b->Alloc(&b->encrypted); + b->encrypted->Initialize(Alignment(), head->size()); + MaybeFlush(*b->encrypted); + if (choices.madvise) b->encrypted->MadviseDontNeed(); + + auto s = cpu_check::Crypto::Encrypt(*head, b->encrypted.get(), + &checksums.crypto_purse); + if (!s.ok()) { + return ReturnError(s.message(), writer_ident); + } + + MaybeFlush(*b->encrypted); + head = b->encrypted.get(); + } + + // Make a copy. + b->pre_copied = head; + if (!b->copied) b->Alloc(&b->copied); + b->copied->Initialize(Alignment(), head->size()); + MaybeFlush(*b->copied); + if (choices.madvise) b->copied->MadviseDontNeed(); + std::string syndrome = b->copied->CopyFrom(*head, choices.copy_method); + + if (!syndrome.empty()) { + return ReturnError( + "writer-detected-copy", + absl::StrCat(JsonRecord("syndrome", syndrome), ", ", writer_ident)); + } + MaybeFlush(*b->copied); + return checksums; +} + +absl::Status Worker::CheckComputations(const std::string &writer_reader_ident, + const Choices &choices, + const Checksums &checksums, + BufferSet *b) { + // Re-verify buffer copy + std::string syndrome = b->copied->Syndrome(*b->pre_copied); + if (!syndrome.empty()) { + return ReturnError("copy", absl::StrCat(JsonRecord("syndrome", syndrome), + ", ", writer_reader_ident)); + } + + MaybeFlush(*b->copied); + MalignBuffer *head = b->copied.get(); + + if (do_encrypt) { + // Decrypt. + if (!b->decrypted) b->Alloc(&b->decrypted); + b->decrypted->Initialize(Alignment(), head->size()); + MaybeFlush(*b->decrypted); + + if (choices.madvise) b->decrypted->MadviseDontNeed(); + auto s = cpu_check::Crypto::Decrypt(*head, checksums.crypto_purse, + b->decrypted.get()); + if (!s.ok()) { + return ReturnError(s.message(), writer_reader_ident); + } + + MaybeFlush(*b->decrypted); + head = b->decrypted.get(); + syndrome = b->pre_encrypted->Syndrome(*head); + if (!syndrome.empty()) { + return ReturnError("decryption_mismatch", + absl::StrCat(JsonRecord("syndrome", syndrome), ", ", + writer_reader_ident)); + } + } + + if (do_compress) { + // Run decompressor. + if (!b->decompressed) b->Alloc(&b->decompressed); + b->decompressed->Initialize(Alignment(), choices.buf_size); + MaybeFlush(*b->decompressed); + + if (choices.madvise) b->decompressed->MadviseDontNeed(); + const auto s = zlib_.Decompress(*head, b->decompressed.get()); + if (!s.ok()) { + return ReturnError("uncompression", + absl::StrCat(Json("syndrome", s.message()), ", ", + writer_reader_ident)); + } + if (b->decompressed->size() != choices.buf_size) { + std::stringstream ss; + ss << "dec_length: " << b->decompressed->size() + << " vs: " << choices.buf_size; + return ReturnError( + "decompressed_size", + absl::StrCat(Json("syndrome", ss.str()), ", ", writer_reader_ident)); + } + MaybeFlush(*b->decompressed); + head = b->decompressed.get(); + } + + if (!b->re_made) b->Alloc(&b->re_made); + b->re_made->Initialize(Alignment(), choices.buf_size); + const cpu_check::FloatingPointResults f_r = pattern_generators_.Generate( + *choices.pattern_generator, choices.hole, round_, choices.copy_method, + choices.use_repstos, choices.exercise_floating_point, b->re_made.get()); + syndrome = b->original->Syndrome(*b->re_made); + + if (!syndrome.empty()) { + return ReturnError("re-make", absl::StrCat(JsonRecord("syndrome", syndrome), + ", ", writer_reader_ident)); + } + + if (checksums.floating_point_results != f_r) { + std::stringstream ss; + ss << "Was: " << checksums.floating_point_results.d << " Is: " << f_r.d; + return ReturnError("fp-double", absl::StrCat(Json("syndrome", ss.str()), + ", ", writer_reader_ident)); + } + + if (do_hashes) { + // Re-run hash func. + const std::string hash = choices.hasher->Hash(*head); + if (checksums.hash_value != hash) { + std::stringstream ss; + ss << "hash was: " << checksums.hash_value << " is: " << hash; + return ReturnError("hash", absl::StrCat(Json("syndrome", ss.str()), ", ", + writer_reader_ident)); + } + } + + auto s = silkscreen_->CheckMySlots(tid_, round_); + if (!s.ok()) { + return ReturnError("Silkscreen", + absl::StrCat(s.message(), ", ", writer_reader_ident)); + } + return absl::OkStatus(); +} + +std::vector Worker::CheckerTids() { + constexpr int kCheckers = 2; + if (!do_hop) { + return std::vector(kCheckers, tid_); + } + std::vector candidates; + for (int i : tid_list_) { + if (i != tid_) candidates.push_back(i); + } + std::shuffle(candidates.begin(), candidates.end(), rndeng_); + std::vector v; + v.reserve(kCheckers); + for (int i = 0; i < kCheckers; i++) { + v.push_back(i < candidates.size() ? candidates[i] : tid_); + } + return v; } void Worker::Run() { - // Array of hash/checksum routines. - typedef struct { - const char *name; - std::string (*func)(const MalignBuffer &); - } hashItem; - std::vector hashers = { - { - "MD5", - [](const MalignBuffer &s) -> std::string { - return OpenSSL_Hash(s, EVP_md5()); - }, - }, - { - "SHA1", - [](const MalignBuffer &s) -> std::string { - return OpenSSL_Hash(s, EVP_sha1()); - }, - }, - { - "SHA256", - [](const MalignBuffer &s) -> std::string { - return OpenSSL_Hash(s, EVP_sha256()); - }, - }, - { - "SHA512", - [](const MalignBuffer &s) -> std::string { - return OpenSSL_Hash(s, EVP_sha512()); - }, - }, - { - "ADLER32", // exported by zlib - [](const MalignBuffer &s) -> std::string { - uLong c = adler32(0, Z_NULL, 0); - c = adler32(c, (const Bytef *)s.data(), s.size()); - return HexData((const char *)&c, sizeof(c)); - }, - }, - { - "CRC32", // exported by zlib. - [](const MalignBuffer &s) -> std::string { - uLong c = crc32(0, Z_NULL, 0); - c = crc32(c, (const Bytef *)s.data(), s.size()); - return HexData((const char *)&c, sizeof(c)); - }, - }, - { - "CRC32C", // crc32 instruction on SSSE3 - [](const MalignBuffer &s) -> std::string { - uint32_t c = crc32c(s.data(), s.size()); - return HexData((const char *)&c, sizeof(c)); - }, - }, - { - "FarmHash64", // Google farmhash - [](const MalignBuffer &s) -> std::string { - uint64_t c = util::Hash64(s.data(), s.size()); - return HexData((const char *)&c, sizeof(c)); - }, - }, - }; - - // Array of compression routines. - typedef struct { - const char *name; - int (*enc)(MalignBuffer *, const MalignBuffer &); - int (*dec)(MalignBuffer *, const MalignBuffer &); - } compressorItem; - std::vector compressors = { - { - "ZLIB", - [](MalignBuffer *o, const MalignBuffer &s) { - uLongf olen = compressBound(s.size()); - o->resize(olen); - int err = compress2((Bytef *)o->data(), &olen, (Bytef *)s.data(), - s.size(), Z_BEST_SPEED); - if (err != Z_OK) { - LOG(DEBUG) << "zlib compression failed: " << err - << " srclen: " << s.size() - << " destlen: " << o->size(); - return err; - } - o->resize(olen); - return 0; - }, - [](MalignBuffer *o, const MalignBuffer &s) { - uLongf olen = o->size(); - int err = uncompress((Bytef *)o->data(), &olen, (Bytef *)s.data(), - s.size()); - if (err != Z_OK) { - LOG(DEBUG) << "zlib decompression failed: " << err - << " srclen: " << s.size() - << " destlen: " << o->size(); - return err; - } - o->resize(olen); - return 0; - }, - }, - }; - - // Choose generator and compressor uniformly. - std::uniform_int_distribution gen_dist( - 0, do_provenance ? 0 : kGenerators.size() - 1); - auto Gen = std::bind(gen_dist, rndeng_); - std::uniform_int_distribution comp_dist(0, compressors.size() - 1); - auto Comp = std::bind(comp_dist, rndeng_); - - // Run one randomly-chosen hash routine each round. - size_t hash_choice; - std::string hash_value; - - EVP_CIPHER_CTX *cipher_ctx; - cipher_ctx = EVP_CIPHER_CTX_new(); + const double t0 = TimeInSeconds(); // Creates FVT controller if we can do so. if (do_fvt) { @@ -1354,22 +677,20 @@ void Worker::Run() { // MalignBuffers are allocated once if !do_madvise. Otherwise they are // reallocated each iteration of the main loop, creating much more memory // allocator work, which may itself exhibit, and suffer from, CPU defects. - std::unique_ptr b; + std::unique_ptr b = std::make_unique(); - uint64_t expected_slot_count = 0; - - Avx avx; - - while (!exiting) { + while (!stopper_->Expired()) { if (std::thread::hardware_concurrency() > 1) { if (!SetAffinity(tid_)) { LOG(WARN) << "Couldnt run on " << tid_ << " sleeping a bit"; - sleep(30); + stopper_->BoundedSleep(30); continue; } } round_++; - if (!b) { + + if (do_madvise) { + // Release and reallocate MalignBuffers. b.reset(new BufferSet); } @@ -1383,17 +704,13 @@ void Worker::Run() { fvt_controller_->MonitorFrequency(); } - auto Turbo = [&turbo_mhz](){ - return Json("turbo", turbo_mhz); - }; + auto Turbo = [&turbo_mhz]() { return Json("turbo", turbo_mhz); }; auto Tid = [this](int tid) { return "{ " + Json("tid", tid) + (do_fvt ? ", " + FVT() : "") + " }"; }; - auto Writer = [this, Tid](){ - return "\"writer\": " + Tid(tid_); - }; + auto Writer = [this, Tid]() { return "\"writer\": " + Tid(tid_); }; LOG_EVERY_N_SECS(INFO, 30) << Jstat( Json("elapsed_s", static_cast(TimeInSeconds() - t0)) + ", " + @@ -1404,395 +721,62 @@ void Worker::Run() { ", " + Json("maxFreq", fvt_controller_->max_mHz()) : "")); - if (do_avx_heavy) { - const std::string e = avx.MaybeGoHot(); - if (!e.empty()) { - LOG(ERROR) << Jfail(e, Writer() + ", " + Turbo()); - errorCount++; - } - } + const Choices choices = MakeChoices(b.get()); + const std::string writer_ident = + absl::StrCat(Writer(), ", ", Turbo(), ", ", choices.summary); -#ifdef USE_BORINGSSL - if (do_ssl_self_check && BORINGSSL_self_test() == 0) { - LOG(ERROR) << Jfail("BORINGSSL_self_test", Writer() + ", " + Turbo()); + auto s = DoComputations(writer_ident, choices, b.get()); + if (!s.ok()) { + LOG(ERROR) << s.status().message(); + LOG(ERROR) << Suspect(tid_); errorCount++; continue; } -#endif + const Checksums checksums = s.value(); - const bool madvise = - do_madvise && std::uniform_int_distribution(0, 1)(rndeng_); - - const size_t bufsize = - std::uniform_int_distribution(kBufMin, kBufMax)(rndeng_); - auto &gen = kGenerators[Gen()]; - auto &comp = compressors[Comp()]; - - const MalignBuffer::PunchedHole hole = PunchedHole(bufsize); - - const MalignBuffer::CopyMethod copy_method = CopyMethod(); - - const bool use_repstos = - do_repstosb && std::uniform_int_distribution(0, 1)(rndeng_); - - auto BlockSummary = [&]() { - std::stringstream block_summary; - block_summary - << Json("pattern", gen.name) << ", " - << Json("copy", MalignBuffer::ToString(copy_method)) << ", " - << Json("memset", use_repstos ? "rep;sto" : "memset") << ", " - << JsonBool("madvise", madvise) << ", " - << Json("size", bufsize) << ", " - << Json("pid", pid_) << ", " - << Json("round", round_) << ", " - << hole.ToString(); - return block_summary.str(); - }; - - auto WriterInfo = [&](){ - return Writer() + ", " + Turbo() + ", " + BlockSummary(); - }; - - if (round_ > 1) { - uint64_t slots_read = 0; - uint64_t errs_this_round = 0; - const uint64_t kErrLimit = 20; - for (size_t k = 0; k < Silkscreen::kSize; k++) { - const std::string err = silkscreen_->CheckMySlot(tid_, round_ - 1, k); - if (!err.empty()) { - slots_read++; - if (err != "=") { - errs_this_round++; - if (errs_this_round <= kErrLimit) { - errorCount++; - LOG(ERROR) << Jfail("Silkscreen", JsonRecord("syndrome", err) + - ", " + WriterInfo()); - } else { - // When Silkscreen fails, it often fails, on several bad machines, - // in a surprising way: all slots owned by reading tid are - // are corrupt, in a way that suggests the previous round's - // writes never happened. Weird, and deserves some study, but - // meanwhile the log spew is suppressed. - } - } - } + // Check the computation. Twice if the first check fails. + const std::vector checker_tids = CheckerTids(); + std::vector failing_tids; + for (int c : checker_tids) { + int newcpu = c; + if (!SetAffinity(newcpu)) { + // Tough luck, can't run on chosen CPU. + // Validate on same cpu we were on. + newcpu = tid_; } - if (errs_this_round > kErrLimit) { - LOG(ERROR) << Jfail( - "Silkscreen", - JsonRecord("syndrome", Json("many_errors", errs_this_round) + ", " + - Json("slots_read", slots_read)) + - ", " + WriterInfo()); - errorCount++; - } - if (expected_slot_count != slots_read) { - LOG(ERROR) << Jfail("Silkscreen", - Json("read", slots_read) + ", " + - Json("expected", expected_slot_count) + ", " + - WriterInfo()); + + auto Reader = [&Tid, &newcpu]() { return "\"reader\": " + Tid(newcpu); }; + const std::string writer_reader_ident = absl::StrCat( + Writer(), ", ", Reader(), ", ", Turbo(), ", ", choices.summary); + + const absl::Status check_status = + CheckComputations(writer_reader_ident, choices, checksums, b.get()); + if (check_status.ok()) { + // It suffices to check just once if the checker confirms that + // computation was correct. + break; + } else { + failing_tids.push_back(newcpu); + LOG(ERROR) << check_status.message(); errorCount++; } } - const uint64_t slots_written = silkscreen_->WriteMySlots(tid_, round_); - if (!expected_slot_count) { - expected_slot_count = slots_written; - } - if (expected_slot_count != slots_written) { - LOG(ERROR) << Jfail("Silkscreen", - Json("written", slots_written) + ", " + - Json("expected", expected_slot_count) + ", " + - WriterInfo()); - errorCount++; - } - - if (do_avx_heavy) { - // If we tried to do AVX heavy stuff. Try to run AVX heavy again to try - // to spike current. - const std::string e = avx.BurnIfAvxHeavy(); - if (!e.empty()) { - LOG(ERROR) << Jfail(e, Writer() + ", " + Turbo()); - errorCount++; + if (!failing_tids.empty()) { + // Guess which LPU is the most likely culprit. The guess is pretty good + // for low failure rate LPUs that haven't corrupted crucial common state. + if (failing_tids.size() > 1) { + // Both checkers think the computation was wrong, likely culprit is the + // writer. + LOG(ERROR) << Suspect(tid_); + } else { + // Only one checker thinks the computation was wrong. Likely he's the + // culprit since the other checker and the writer agree. + LOG(ERROR) << Suspect(failing_tids[0]); } - } - - const auto buffer_seed = rndeng_(); - if (!b->original) b->Alloc(&b->original); - b->original->Initialize(Alignment(), bufsize); - const FloatingPointResults f = - GenerateData(gen, hole, buffer_seed, - copy_method, use_repstos, b->original.get()); - MaybeFlush(*b->original); - - MalignBuffer* head = b->original.get(); - - if (do_hashes) { - hash_choice = - std::uniform_int_distribution(0, hashers.size() - 1)(rndeng_); - hash_value = hashers[hash_choice].func(*head); - } - - if (do_compress) { - // Run our randomly chosen compressor. - if (!b->compressed) b->Alloc(&b->compressed); - b->compressed->Initialize(Alignment(), bufsize); - MaybeFlush(*b->compressed); - - const int err = comp.enc(b->compressed.get(), *head); - LOG(DEBUG) << WriterInfo() - << " original->size(): " << head->size() - << ", compressed.size(): " << b->compressed->size() << "."; - if (err) { - LOG(ERROR) << Jfail("Compression", - Json("syndrome", err) + ", " + WriterInfo()); - errorCount++; - continue; - } - MaybeFlush(*b->compressed); - head = b->compressed.get(); - LOG(DEBUG) << WriterInfo() << "compress done."; - if (exiting) break; - } - - const unsigned char key[33] = "0123456789abcdef0123456789abcdef"; - const std::string ivec(b->original->data(), kBufMin); - unsigned char gmac[16]; - - const MalignBuffer* const unencrypted = head; - if (do_encrypt) { - // Encrypt. - if (!b->encrypted) b->Alloc(&b->encrypted); - b->encrypted->Initialize(Alignment(), head->size()); - MaybeFlush(*b->encrypted); - int enc_len = 0, enc_unused_len = 0; - EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key, - (unsigned char *)ivec.data(), 1); - - if (madvise) MadviseDontNeed(*b->encrypted); - if (EVP_CipherUpdate( - cipher_ctx, (unsigned char *)b->encrypted->data(), &enc_len, - (unsigned char *)head->data(), head->size()) != 1) { - LOG(ERROR) << Jfail("EVP_CipherUpdate", WriterInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - if (EVP_CipherFinal_ex(cipher_ctx, nullptr, &enc_unused_len) != 1) { - LOG(ERROR) << Jfail("encrypt_EVP_CipherFinal_ex", WriterInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - enc_len += enc_unused_len; - if (enc_len != (int)b->encrypted->size()) { - std::stringstream ss; - ss << "enc_length: " << enc_len << " vs: " << b->encrypted->size(); - LOG(ERROR) << Jfail("encrypt_length_mismatch", - Json("syndrome", ss.str()) + ", " + WriterInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_GET_TAG, sizeof(gmac), - gmac) != 1) { - LOG(ERROR) << Jfail("EVP_CTRL_GCM_GET_TAG", WriterInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - EVP_CIPHER_CTX_cleanup(cipher_ctx); - MaybeFlush(*b->encrypted); - head = b->encrypted.get(); - LOG(DEBUG) << "Encrypt done " << WriterInfo(); - if (exiting) break; - } - - // Make a copy. - if (!b->copied) b->Alloc(&b->copied); - b->copied->Initialize(Alignment(), head->size()); - MaybeFlush(*b->copied); - if (madvise) MadviseDontNeed(*b->copied); - std::string syndrome = b->copied->CopyFrom(*head, copy_method); - - if (!syndrome.empty()) { - LOG(ERROR) << Jfail( - "writer-detected-copy", - JsonRecord("syndrome", syndrome) + ", " + WriterInfo()); - errorCount++; continue; } - MaybeFlush(*b->copied); - - // Switch to an alternate CPU. - - int newcpu = tid_; - if (do_hop && std::thread::hardware_concurrency() > 1) { - std::vector cpus; - for (int i : tid_list_) { - if (i == tid_) continue; - cpus.push_back(i); - } - if (!cpus.empty()) { - int cpuoff = - std::uniform_int_distribution(0, cpus.size() - 1)(rndeng_); - newcpu = cpus[cpuoff]; - cpus.erase(cpus.begin() + cpuoff); - if (!SetAffinity(newcpu)) { - // Tough luck, can't run on chosen CPU. - // Validate on same cpu we were on. - newcpu = tid_; - } - } - } - - auto Reader = [&Tid, &newcpu](){ - return "\"reader\": " + Tid(newcpu); - }; - auto WriterReaderInfo = [&](){ - return - Writer() + ", " + Reader() + ", " + Turbo() + ", " + BlockSummary(); - }; - - // Re-verify buffer copy - syndrome = b->copied->Syndrome(*head); - if (!syndrome.empty()) { - LOG(ERROR) << Jfail( - "copy", JsonRecord("syndrome", syndrome) + ", " + WriterInfo()); - errorCount++; - continue; - } - - MaybeFlush(*b->copied); - head = b->copied.get(); - - if (do_encrypt) { - // Decrypt. - if (!b->decrypted) b->Alloc(&b->decrypted); - b->decrypted->Initialize(Alignment(), head->size()); - MaybeFlush(*b->decrypted); - - int dec_len = 0, dec_extra_len = 0; - EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key, - (unsigned char *)ivec.data(), 0); - if (madvise) MadviseDontNeed(*b->decrypted); - if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_SET_TAG, sizeof(gmac), - gmac) != 1) { - LOG(ERROR) << Jfail("EVP_CTRL_GCM_SET_TAG", WriterReaderInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - if (EVP_CipherUpdate( - cipher_ctx, (unsigned char *)b->decrypted->data(), &dec_len, - (unsigned char *)head->data(), head->size()) != 1) { - LOG(ERROR) << Jfail("decryption", WriterReaderInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - if (EVP_CipherFinal_ex( - cipher_ctx, (unsigned char *)(b->decrypted->data() + dec_len), - &dec_extra_len) != 1) { - LOG(ERROR) << Jfail("decrypt_EVP_CipherFinal_ex", WriterReaderInfo()); - errorCount++; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - continue; - } - dec_len += dec_extra_len; - EVP_CIPHER_CTX_cleanup(cipher_ctx); - if (dec_len != (int)b->decrypted->size()) { - std::stringstream ss; - ss << "dec_length: " << dec_len << " vs: " << b->decrypted->size(); - LOG(ERROR) << Jfail("decrypt_length_mismatch", - Json("syndrome", ss.str()) + ", " + - WriterReaderInfo()); - errorCount++; - continue; - } - MaybeFlush(*b->decrypted); - head = b->decrypted.get(); - syndrome = unencrypted->Syndrome(*head); - if (!syndrome.empty()) { - LOG(ERROR) << Jfail( - "decryption_mismatch", - JsonRecord("syndrome", syndrome) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - LOG(DEBUG) << WriterReaderInfo() << " decrypt done"; - if (exiting) break; - } - - if (do_compress) { - // Run decompressor. - if (!b->decompressed) b->Alloc(&b->decompressed); - b->decompressed->Initialize(Alignment(), bufsize); - MaybeFlush(*b->decompressed); - - if (madvise) MadviseDontNeed(*b->decompressed); - const int err = comp.dec(b->decompressed.get(), *head); - if (err) { - LOG(ERROR) << Jfail("uncompression", - Json("syndrome", err) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - if (b->decompressed->size() != bufsize) { - std::stringstream ss; - ss << "dec_length: " << b->decompressed->size() << " vs: " << bufsize; - LOG(ERROR) << Jfail( - "decompressed_size", - Json("syndrome", ss.str()) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - MaybeFlush(*b->decompressed); - head = b->decompressed.get(); - LOG(DEBUG) << WriterReaderInfo() << " uncompress done"; - } - - if (!b->re_made) b->Alloc(&b->re_made); - b->re_made->Initialize(Alignment(), bufsize); - const FloatingPointResults f_r = - GenerateData(gen, hole, buffer_seed, - copy_method, use_repstos, b->re_made.get()); - syndrome = b->original->Syndrome(*b->re_made); - - if (!syndrome.empty()) { - LOG(ERROR) << Jfail("re-make", JsonRecord("syndrome", syndrome) + ", " + - WriterReaderInfo()); - errorCount++; - continue; - } - - if (f != f_r) { - std::stringstream ss; - ss << "Was: " << f.d << " Is: " << f_r.d; - LOG(ERROR) << Jfail( - "fp-double", Json("syndrome", ss.str()) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - - if (do_hashes) { - // Re-run hash func. - std::string hash = hashers[hash_choice].func(*head); - if (hash_value != hash) { - std::stringstream ss; - ss << "hash was: " << hash_value << " is: " << hash; - LOG(ERROR) << Jfail( - "hash", Json("syndrome", ss.str()) + ", " + WriterReaderInfo()); - errorCount++; - continue; - } - LOG(DEBUG) << WriterReaderInfo() << " rehash done"; - } - - // Release MalignBuffer memory allocations. - if (do_madvise) b.reset(nullptr); - successCount++; } - EVP_CIPHER_CTX_free(cipher_ctx); LOG(INFO) << "tid " << tid_ << " exiting."; } @@ -1833,12 +817,10 @@ int main(int argc, char **argv) { std::vector tid_list; int64_t timeout = 0; -#ifdef IN_GOOGLE3 // Initialize the symbolizer to get a human-readable stack trace. absl::InitializeSymbolizer(argv[0]); absl::FailureSignalHandlerOptions options; absl::InstallFailureSignalHandler(options); -#endif for (int i = 1; i < argc; i++) { const char *flag = argv[i]; @@ -1851,39 +833,35 @@ int main(int argc, char **argv) { case 'b': do_ssl_self_check = false; break; - case 'c': - { - std::string c = ""; - for (flag++; *flag != 0; flag++) { - c += *flag; - } - std::stringstream s(c); - int t; - while (s >> t) { - tid_list.push_back(t); - if (s.peek() == ',') s.ignore(); - } + case 'c': { + std::string c = ""; + for (flag++; *flag != 0; flag++) { + c += *flag; } - break; + std::stringstream s(c); + int t; + while (s >> t) { + tid_list.push_back(t); + if (s.peek() == ',') s.ignore(); + } + } break; case 'd': do_repstosb = false; break; case 'e': do_encrypt = false; break; - case 'f': - { - std::string c(++flag); - flag += c.length(); - std::stringstream s(c); - s >> fixed_min_frequency; - fixed_max_frequency = fixed_min_frequency; - if (s.get() == '-') { - s >> fixed_max_frequency; - do_freq_sweep = true; - } + case 'f': { + std::string c(++flag); + flag += c.length(); + std::stringstream s(c); + s >> fixed_min_frequency; + fixed_max_frequency = fixed_min_frequency; + if (s.get() == '-') { + s >> fixed_max_frequency; + do_freq_sweep = true; } - break; + } break; case 'F': do_flush = true; break; @@ -1920,14 +898,12 @@ int main(int argc, char **argv) { do_hashes = false; do_compress = false; break; - case 'q': - { - std::string c(++flag); - flag += c.length(); - std::stringstream s(c); - s >> error_limit; - } - break; + case 'q': { + std::string c(++flag); + flag += c.length(); + std::stringstream s(c); + s >> error_limit; + } break; case 'r': do_repmovsb = false; break; @@ -1946,22 +922,18 @@ int main(int argc, char **argv) { case 'Y': do_freq_sweep = true; break; - case 'k': - { - std::string c(++flag); - flag += c.length(); - std::stringstream s(c); - s >> seconds_per_freq; - } - break; - case 't': - { - std::string c(++flag); - flag += c.length(); - std::stringstream s(c); - s >> timeout; - } - break; + case 'k': { + std::string c(++flag); + flag += c.length(); + std::stringstream s(c); + s >> seconds_per_freq; + } break; + case 't': { + std::string c(++flag); + flag += c.length(); + std::stringstream s(c); + s >> timeout; + } break; case 'z': do_compress = false; break; @@ -1995,11 +967,7 @@ int main(int argc, char **argv) { std::vector threads; std::vector workers; - std::vector words = ReadDict(); - if (words.empty()) { - LOG(ERROR) << "No word list found."; - exit(1); - } + int cpus = std::thread::hardware_concurrency(); LOG(INFO) << "Detected hardware concurrency: " << cpus; @@ -2024,26 +992,28 @@ int main(int argc, char **argv) { } } + const double t0 = TimeInSeconds(); // Silkscreen instance shared by all threads. - Silkscreen silkscreen(tid_list); + cpu_check::Silkscreen silkscreen(tid_list); + + static Stopper stopper(timeout); // Shared by all threads for (int tid : tid_list) { - workers.push_back(new Worker(getpid(), &words, tid_list, tid, &silkscreen)); + workers.push_back( + new Worker(getpid(), tid_list, tid, &silkscreen, &stopper)); threads.push_back(new std::thread(&Worker::Run, workers.back())); } - signal(SIGTERM, [](int) { exiting = true; }); - signal(SIGINT, [](int) { exiting = true; }); + + signal(SIGTERM, [](int) { stopper.Stop(); }); + signal(SIGINT, [](int) { stopper.Stop(); }); struct timeval last_cpu = {0, 0}; double last_time = t0; - while (!exiting) { - sleep(60); + while (!stopper.Expired()) { + stopper.BoundedSleep(60); struct rusage ru; double secs = TimeInSeconds(); - if (timeout > 0 && secs >= t0 + timeout) { - exiting = true; - } double secondsPerError = (secs - t0) / errorCount.load(); if (getrusage(RUSAGE_SELF, &ru) == -1) { LOG(ERROR) << "getrusage failed: " << strerror(errno); @@ -2052,8 +1022,8 @@ int main(int argc, char **argv) { (ru.ru_utime.tv_usec - last_cpu.tv_usec)) / 1000000.0; LOG(INFO) << "Errors: " << errorCount.load() - << " Successes: " << successCount.load() - << " CPU " << cpu / (secs - last_time) << " s/s" + << " Successes: " << successCount.load() << " CPU " + << cpu / (secs - last_time) << " s/s" << " Seconds Per Error: " << secondsPerError; last_cpu = ru.ru_utime; } diff --git a/crypto.cc b/crypto.cc new file mode 100644 index 0000000..c8dd282 --- /dev/null +++ b/crypto.cc @@ -0,0 +1,112 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "crypto.h" + +#include "config.h" +#include "absl/status/status.h" + +namespace cpu_check { + +namespace { +constexpr unsigned char key[33] = "0123456789abcdef0123456789abcdef"; +}; // namespace + +absl::Status Crypto::Encrypt(const MalignBuffer &plain_text, + MalignBuffer *cipher_text, CryptoPurse *purse) { + memset(purse->i_vec, 0, sizeof(purse->i_vec)); + memcpy(purse->i_vec, plain_text.data(), + std::min(plain_text.size(), sizeof(purse->i_vec))); + + int enc_len = 0; + int enc_unused_len = 0; + EVP_CIPHER_CTX *cipher_ctx = EVP_CIPHER_CTX_new(); + + EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key, purse->i_vec, 1); + if (EVP_CipherUpdate( + cipher_ctx, + reinterpret_cast(cipher_text->data()), + &enc_len, reinterpret_cast(plain_text.data()), + plain_text.size()) != 1) { + return ReturnError("EVP_CipherUpdate", cipher_ctx); + } + if (EVP_CipherFinal_ex(cipher_ctx, nullptr, &enc_unused_len) != 1) { + return ReturnError("encrypt_EVP_CipherFinal_ex", cipher_ctx); + } + enc_len += enc_unused_len; + if (enc_len != (int)cipher_text->size()) { + return ReturnError("encrypt_length_mismatch", cipher_ctx); + } + if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_GET_TAG, + sizeof(purse->gmac_tag), purse->gmac_tag) != 1) { + return ReturnError("EVP_CTRL_GCM_GET_TAG", cipher_ctx); + } + EVP_CIPHER_CTX_free(cipher_ctx); + return absl::OkStatus(); +} + +absl::Status Crypto::Decrypt(const MalignBuffer &cipher_text, + const CryptoPurse &purse, + MalignBuffer *plain_text) { + int dec_len = 0; + int dec_extra_len = 0; + EVP_CIPHER_CTX *cipher_ctx = EVP_CIPHER_CTX_new(); + + EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key, purse.i_vec, 0); + + // Make a non-const copy of gmac_tag because that's what EVP_CIPHER_CTX_ctrl + // requires, even though it won't be modified in this use. + unsigned char copied_tag[sizeof(purse.gmac_tag)]; + memcpy(copied_tag, purse.gmac_tag, sizeof(purse.gmac_tag)); + + if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_SET_TAG, sizeof(copied_tag), + reinterpret_cast(copied_tag)) != 1) { + return ReturnError("EVP_CTRL_GCM_SET_TAG", cipher_ctx); + } + if (EVP_CipherUpdate( + cipher_ctx, reinterpret_cast(plain_text->data()), + &dec_len, reinterpret_cast(cipher_text.data()), + cipher_text.size()) != 1) { + return ReturnError("Decryption", cipher_ctx); + } + if (EVP_CipherFinal_ex( + cipher_ctx, + reinterpret_cast(plain_text->data() + dec_len), + &dec_extra_len) != 1) { + return ReturnError("decrypt_EVP_CipherFinal_ex", cipher_ctx); + } + dec_len += dec_extra_len; + if (dec_len != (int)plain_text->size()) { + return ReturnError("decrypt_length_mismatch", cipher_ctx); + } + EVP_CIPHER_CTX_free(cipher_ctx); + return absl::OkStatus(); +} + +absl::Status Crypto::SelfTest() { +#ifdef USE_BORINGSSL + if (BORINGSSL_self_test() == 0) { + return absl::Status(absl::StatusCode::kInternal, "BORINGSSL_self_test"); + } +#endif + + return absl::OkStatus(); +} + +absl::Status Crypto::ReturnError(absl::string_view message, + EVP_CIPHER_CTX *cipher_ctx) { + EVP_CIPHER_CTX_free(cipher_ctx); + return absl::Status(absl::StatusCode::kInternal, message); +} +}; // namespace cpu_check diff --git a/crypto.h b/crypto.h new file mode 100644 index 0000000..f92c5d4 --- /dev/null +++ b/crypto.h @@ -0,0 +1,54 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_CRYPTO_H_ +#define THIRD_PARTY_CPU_CHECK_CRYPTO_H_ + +#include "malign_buffer.h" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include +#include + +namespace cpu_check { + +class Crypto { + public: + // Encryption produces these values, which are consumed by decryption. + struct CryptoPurse { + unsigned char i_vec[12]; + unsigned char gmac_tag[16]; + }; + + // Encrypts 'plain_text' to 'cipher_text' and stores i_vec and gmac + // in 'purse'. + static absl::Status Encrypt(const MalignBuffer &plain_text, + MalignBuffer *cipher_text, CryptoPurse *purse); + + // Decrypts 'cipher_text' into 'plain_text' using i_vec and gmac from 'purse'. + static absl::Status Decrypt(const MalignBuffer &cipher_text, + const CryptoPurse &purse, + MalignBuffer *plain_text); + + // Runs crypto self test, if available. + static absl::Status SelfTest(); + + private: + // Returns kInternal error and frees context 'cipher_ctx'. + static absl::Status ReturnError(absl::string_view message, + EVP_CIPHER_CTX *cipher_ctx); +}; + +}; // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_CRYPTO_H_ diff --git a/hasher.cc b/hasher.cc new file mode 100644 index 0000000..fc25cf4 --- /dev/null +++ b/hasher.cc @@ -0,0 +1,95 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hasher.h" + +#include "crc32c.h" +#include "utils.h" +#include "third_party/farmhash/src/farmhash.h" +#include +#include +#include + +namespace cpu_check { +namespace { +std::string OpenSSL_Hash(const MalignBuffer &s, const EVP_MD *type) { + EVP_MD_CTX *ctx; + ctx = EVP_MD_CTX_create(); + EVP_DigestInit_ex(ctx, type, nullptr); + std::string hash; + hash.resize(EVP_MD_CTX_size(ctx)); + MalignBuffer::InitializeMemoryForSanitizer(hash.data(), EVP_MD_CTX_size(ctx)); + EVP_DigestUpdate(ctx, s.data(), s.size()); + EVP_DigestFinal_ex(ctx, (uint8_t *)&hash[0], nullptr); + EVP_MD_CTX_destroy(ctx); + return HexStr(hash); +} +} // namespace + +std::string Md5::Hash(const MalignBuffer &b) const { + return OpenSSL_Hash(b, EVP_md5()); +} + +std::string Sha1::Hash(const MalignBuffer &b) const { + return OpenSSL_Hash(b, EVP_sha1()); +} + +std::string Sha256::Hash(const MalignBuffer &b) const { + return OpenSSL_Hash(b, EVP_sha256()); +} + +std::string Sha512::Hash(const MalignBuffer &b) const { + return OpenSSL_Hash(b, EVP_sha512()); +} + +std::string Adler32::Hash(const MalignBuffer &b) const { + uLong c = adler32(0, Z_NULL, 0); + c = adler32(c, reinterpret_cast(b.data()), b.size()); + return HexData(reinterpret_cast(&c), sizeof(c)); +} + +std::string Crc32::Hash(const MalignBuffer &b) const { + uLong c = crc32(0, Z_NULL, 0); + c = crc32(c, reinterpret_cast(b.data()), b.size()); + return HexData(reinterpret_cast(&c), sizeof(c)); +} + +std::string Crc32C::Hash(const MalignBuffer &b) const { + const uint32_t c = crc32c(b.data(), b.size()); + return HexData(reinterpret_cast(&c), sizeof(c)); +} + +std::string FarmHash64::Hash(const MalignBuffer &b) const { + const uint64_t c = util::Hash64(b.data(), b.size()); + return HexData(reinterpret_cast(&c), sizeof(c)); +} + +Hashers::Hashers() { + hashers_.emplace_back(new Md5); + hashers_.emplace_back(new Sha1); + hashers_.emplace_back(new Sha256); + hashers_.emplace_back(new Sha512); + hashers_.emplace_back(new Adler32); + hashers_.emplace_back(new Crc32); + hashers_.emplace_back(new Crc32C); + hashers_.emplace_back(new FarmHash64); +} + +const Hasher &Hashers::RandomHasher(uint64_t seed) const { + std::knuth_b rng(seed); + const size_t k = + std::uniform_int_distribution(0, hashers_.size() - 1)(rng); + return *hashers_[k]; +} +} // namespace cpu_check diff --git a/hasher.h b/hasher.h new file mode 100644 index 0000000..8d7ff53 --- /dev/null +++ b/hasher.h @@ -0,0 +1,96 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_HASH_H_ +#define THIRD_PARTY_CPU_CHECK_HASH_H_ + +#include +#include +#include + +#include "malign_buffer.h" + +namespace cpu_check { + +class Hasher { + public: + virtual ~Hasher() {} + virtual std::string Name() const = 0; + virtual std::string Hash(const MalignBuffer &b) const = 0; +}; + +class Md5 : public Hasher { + public: + std::string Name() const override { return "MD5"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Sha1 : public Hasher { + public: + std::string Name() const override { return "SHA1"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Sha256 : public Hasher { + public: + std::string Name() const override { return "SHA256"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Sha512 : public Hasher { + public: + std::string Name() const override { return "SHA512"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Adler32 : public Hasher { + public: + std::string Name() const override { return "ADLER32"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Crc32 : public Hasher { + public: + std::string Name() const override { return "CRC32"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Crc32C : public Hasher { + public: + std::string Name() const override { return "CRC32C"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class FarmHash64 : public Hasher { + public: + std::string Name() const override { return "FarmHash64"; } + std::string Hash(const MalignBuffer &b) const override; +}; + +class Hashers { + public: + Hashers(); + + // Returns a randomly selected hasher. + const Hasher &RandomHasher(uint64_t seed) const; + + const std::vector> &hashers() const { + return hashers_; + } + + private: + std::vector> hashers_; +}; +} // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_HASH_H_ diff --git a/malign_buffer.cc b/malign_buffer.cc new file mode 100644 index 0000000..5845ffb --- /dev/null +++ b/malign_buffer.cc @@ -0,0 +1,382 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "malign_buffer.h" + +#include + +#if defined(__i386__) || defined(__x86_64__) +#include +#endif + +#include +#include + +#include +#include +#include + +#include "log.h" +#include "utils.h" + +#undef HAS_FEATURE_MEMORY_SANITIZER +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#define HAS_FEATURE_MEMORY_SANITIZER +#endif +#endif + +#if defined(__i386__) || defined(__x86_64__) +#define X86_TARGET_ATTRIBUTE(s) __attribute__((target(s))) +#else +#define X86_TARGET_ATTRIBUTE(s) +#endif + +namespace cpu_check { +namespace { + +inline void __movsb(char *dst, const char *src, size_t size) { +#if defined(__i386__) || defined(__x86_64__) + __asm__ __volatile__("rep movsb" + : "+D"(dst), "+S"(src), "+c"(size) + : + : "memory"); +#else + LOG(FATAL) << "Cannot rep;movsb"; +#endif +} + +inline void __stosb(void *dst, unsigned char c, size_t size) { +#if defined(__i386__) || defined(__x86_64__) + __asm__ __volatile__("rep stosb" : "+D"(dst), "+c"(size) : "a"(c) : "memory"); +#else + LOG(FATAL) << "Cannot rep;stosb"; +#endif +} + +inline void __sse_128_memcpy(char *dst, const char *src, size_t size) { +#if (defined(__i386__) || defined(__x86_64__)) + size_t blks = size / 16; + for (int i = 0; i < blks; i++) { + _mm_storeu_si128( + reinterpret_cast<__m128i *>(dst) + i, + _mm_loadu_si128(reinterpret_cast(src) + i)); + } + memcpy(dst + blks * 16, src + blks * 16, size - blks * 16); +#else + LOG(FATAL) << "SSE not available"; +#endif +} + +X86_TARGET_ATTRIBUTE("avx") +inline void __avx_256_memcpy(char *dst, const char *src, size_t size) { +#if (defined(__i386__) || defined(__x86_64__)) + size_t blks = size / 32; + for (int i = 0; i < blks; i++) { + _mm256_storeu_si256( + reinterpret_cast<__m256i *>(dst) + i, + _mm256_loadu_si256(reinterpret_cast(src) + i)); + } + memcpy(dst + blks * 32, src + blks * 32, size - blks * 32); +#else + LOG(FATAL) << "x86 only"; +#endif +} + +X86_TARGET_ATTRIBUTE("avx512f") +inline void __avx_512_memcpy(char *dst, const char *src, size_t size) { +#if (defined(__i386__) || defined(__x86_64__)) + size_t blks = size / 64; + for (int i = 0; i < blks; i++) { + _mm512_storeu_si512( + reinterpret_cast<__m512i *>(dst) + i, + _mm512_loadu_si512(reinterpret_cast(src) + i)); + } + memcpy(dst + blks * 64, src + blks * 64, size - blks * 64); +#else + LOG(FATAL) << "x86 only"; +#endif +} +} // namespace + +size_t MalignBuffer::RoundUpToPageSize(size_t k) { + return ((k + kPageSize - 1) / kPageSize) * kPageSize; +} + +// Helper to make MSAN happy. NOP if memory sanitizer is not enabled. +void MalignBuffer::InitializeMemoryForSanitizer(char *addr, size_t size) { +#ifdef HAS_FEATURE_MEMORY_SANITIZER + std::default_random_engine rnd; + std::uniform_int_distribution dist(std::numeric_limits::min(), + std::numeric_limits::max()); + for (size_t i = 0; i < size; i++) { + addr[i] = dist(rnd); + } +#endif +} + +const size_t MalignBuffer::kPageSize = sysconf(_SC_PAGESIZE); +const size_t MalignBuffer::kCacheLineSize = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); + +std::string MalignBuffer::ToString(CopyMethod m) { + switch (m) { + case kMemcpy: + return "memcpy"; + case kRepMov: + return "rep;mov"; + case kSseBy128: + return "sse:128"; + case kAvxBy256: + return "avx:256"; + case kAvxBy512: + return "avx:512"; + } +} + +size_t MalignBuffer::RandomAlignment(uint64_t seed) { + std::knuth_b rng(seed); + return std::uniform_int_distribution(0, kPageSize - 1)(rng); +} + +MalignBuffer::MalignBuffer(size_t capacity) + : capacity_(capacity), + base_address_( + aligned_alloc(kPageSize, RoundUpToPageSize(capacity) + kPageSize)) { + if (base_address_ == nullptr) { + LOG(FATAL) << "Failed allocate for capacity: " << capacity; + } + // There are lots of places that use unitialized MalignBuffer. So just + // fill some pseudo-random bytes if cpu_check is compiled with msan. + InitializeMemoryForSanitizer(static_cast(base_address_), capacity_); +} +MalignBuffer::MalignBuffer(size_t alignment_offset, absl::string_view s) + : MalignBuffer(s.size() + alignment_offset) { + Initialize(alignment_offset, s.size()); + CopyFrom(s, kMemcpy); +} + +MalignBuffer::~MalignBuffer() { free(base_address_); } + +void MalignBuffer::Initialize(size_t alignment_offset, size_t length) { + if (length > capacity_) { + LOG(FATAL) << "Length: " << length << " Capacity: " << capacity_; + } + if (alignment_offset >= kPageSize) { + LOG(FATAL) << "Alignment: " << alignment_offset + << " PageSize: " << kPageSize; + } + alignment_offset_ = alignment_offset; + length_ = length; + buffer_address_ = static_cast(base_address_) + alignment_offset_; +} + +void MalignBuffer::resize(size_t length) { + Initialize(alignment_offset_, length); +} + +std::string MalignBuffer::CopyFrom(const MalignBuffer &that, CopyMethod m) { + CopyFrom(absl::string_view(that.data(), that.size()), m); + return Syndrome(that); +} + +void MalignBuffer::CopyFrom(absl::string_view src, CopyMethod m) { + if (size() != src.size()) { + LOG(FATAL) << "this.size: " << size() << " src.size:" << src.size(); + } + CopyFrom(0, src, m); +} + +void MalignBuffer::CopyFrom(size_t pos, absl::string_view src, CopyMethod m) { + if (pos + src.size() > size()) { + LOG(FATAL) << "this.size: " << size() << " src.size:" << src.size() + << " pos: " << pos; + } + switch (m) { + case kMemcpy: + // Assumes memcpy doesn't use rep;movsb; false in lots of environments. + memcpy(data() + pos, src.data(), src.size()); + break; + case kRepMov: + __movsb(data() + pos, src.data(), src.size()); + break; + case kSseBy128: + __sse_128_memcpy(data() + pos, src.data(), src.size()); + break; + case kAvxBy256: + __avx_256_memcpy(data() + pos, src.data(), src.size()); + break; + case kAvxBy512: + __avx_512_memcpy(data() + pos, src.data(), src.size()); + break; + } +} + +std::string MalignBuffer::Syndrome(const MalignBuffer &that) const { + std::stringstream s; + std::string syndrome = CorruptionSyndrome(that); + if (syndrome.empty()) return ""; + s << syndrome << ", \"this\": \"" << static_cast(data()) + << "\", " + << "\"that\": \"" << static_cast(that.data()) << "\""; + return s.str(); +} + +std::string MalignBuffer::CorruptionSyndrome(const MalignBuffer &that) const { + std::stringstream s; + if (size() != that.size()) { + s << Json("unequalSizeThis", size()) << ", " + << Json("unequalSizeThat", that.size()); + return s.str(); + } + bool failed_memcmp = memcmp(data(), that.data(), that.size()); + + int wrong_bytes = 0; + int wrong_bits = 0; + int byte_faults = 0; + int first_wrong = INT_MAX; + int last_wrong = INT_MIN; + std::vector lane_errors(8, 0); + for (size_t i = 0; i < size(); i++) { + unsigned char a = *(data() + i); + unsigned char b = *(that.data() + i); + unsigned char d = a ^ b; + if (d) { + first_wrong = std::min(first_wrong, i); + last_wrong = std::max(last_wrong, i); + byte_faults |= d; + wrong_bytes++; + wrong_bits += __builtin_popcount(d); + for (size_t i = 0; i < 8; i++) { + if ((d >> i) & 1) { + lane_errors[i]++; + } + } + } + } + if (wrong_bits || wrong_bytes) { + const int range_width = (last_wrong - first_wrong) + 1; + s << Json("cmpResult", + (failed_memcmp ? "Failed_Memcmp" : "**Passed_Memcmp**")) + << ", " << Json("wrongByteCount", wrong_bytes) << ", " + << Json("wrongBitCount", wrong_bits) << ", " + << Json("corruptionWidth", range_width) << ", " + << Json("corruptStart", first_wrong) << ", " + << Json("corruptByteBitMask", byte_faults) << ", " + << "\"byBitLane\": ["; + for (size_t i = 0; i < 8; i++) { + if (i) s << ", "; + s << lane_errors[i]; + } + s << " ] "; + // Dump up to 64 corrupted locations. + std::stringstream dump; + dump << " \"byteErrors\": [ " << std::hex; + uint64_t buf_a = 0; + uint64_t buf_b = 0; + for (size_t k = 0; k < std::min(64, range_width); k++) { + uint8_t a = *(data() + first_wrong + k); + uint8_t b = *(that.data() + first_wrong + k); + if (k) dump << ", "; + dump << "[ " << std::setw(2) << "\"0x" << static_cast(a) << "\", " + << std::setw(2) << "\"0x" << static_cast(b) << "\" "; + buf_a = (buf_a >> 8) | static_cast(a) << 56; + buf_b = (buf_b >> 8) | static_cast(b) << 56; + if ((k >= 7) && (7 == ((first_wrong + k) % 8))) { + dump << ", " << CrackId(buf_a) << ", " << CrackId(buf_b); + buf_a = 0; + buf_b = 0; + } + dump << " ]"; + } + dump << " ] "; + return s.str() + ", " + dump.str(); + } else { + if (!failed_memcmp) return ""; + return Json("cmpResult", "**Failed_Memcmp**"); + } +} + +std::string MalignBuffer::CrackId(uint64_t v) const { + std::stringstream s; + s << std::hex << " [\"0x" << std::setw(4) << (v >> 48) << "\", \"0x" + << std::setw(6) << ((v >> 24) & 0xffffff) << "\", \"0x" << std::setw(6) + << (v & 0xffffff) << "\"]"; + return s.str(); +} + +void MalignBuffer::RandomFlush(std::knuth_b *rng) const { +#if defined(__i386__) || defined(__x86_64__) + // Note: no barriers used. + const char *p = buffer_address_ + alignment_offset_; + while (p < buffer_address_ + length_) { + if (std::uniform_int_distribution(0, 1)(*rng)) { + __builtin_ia32_clflush(p); + } + p += kCacheLineSize; + } +#endif +} + +std::string MalignBuffer::PunchedHole::ToString() const { + if (length) { + return JsonRecord("hole", Json("start", start) + ", " + + Json("length", length) + ", " + + Json("v", static_cast(v))); + } else { + return JsonNull("hole"); + } +} + +void MalignBuffer::Memset(size_t offset, unsigned char v, size_t length, + bool use_rep_stos) { + if (use_rep_stos) { + __stosb(data() + offset, v, length); + } else { + memset(data() + offset, v, length); + } +} + +void MalignBuffer::PunchHole(const PunchedHole &hole, bool use_rep_stos) { + if (hole.length) { + Memset(hole.start, hole.v, hole.length, use_rep_stos); + } +} + +// Hints to the OS to release the buffer's memory. +void MalignBuffer::MadviseDontNeed() const { + // Round up the buffer start address to a page boundary. + intptr_t start = ((intptr_t)data() + kPageSize - 1) & ~(kPageSize - 1); + // Round down the buffer end address to a page boundary. + intptr_t end = ((intptr_t)(data() + size() - 1)) & ~(kPageSize - 1); + if (end > start) { + const size_t length = end - start; + if (madvise((char *)start, length, MADV_DONTNEED) == -1) { + LOG(WARN) << "tid " + << " madvise(MADV_DONTNEED) failed: " << strerror(errno) + << " length: " << length; + } + } +} + +MalignBuffer::PunchedHole MalignBuffer::RandomPunchedHole(uint64_t seed) const { + std::knuth_b rng(seed); + MalignBuffer::PunchedHole hole; + hole.length = std::uniform_int_distribution( + 1, std::min(length_, 8192))(rng); + hole.start = + std::uniform_int_distribution(0, length_ - hole.length)(rng); + return hole; +} + +} // namespace cpu_check diff --git a/malign_buffer.h b/malign_buffer.h new file mode 100644 index 0000000..ceee77b --- /dev/null +++ b/malign_buffer.h @@ -0,0 +1,124 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_MALIGN_BUFFER_H_ +#define THIRD_PARTY_CPU_CHECK_MALIGN_BUFFER_H_ + +#include +#include + +#include "absl/strings/string_view.h" + +namespace cpu_check { + +// Data buffer supporting various alignments, copy mechanisms, and verification +// methods. +class MalignBuffer { + public: + struct PunchedHole { + std::string ToString() const; + size_t start = 0; + size_t length = 0; + unsigned char v = 0x53; + }; + + enum CopyMethod { + kMemcpy, + kRepMov, + kSseBy128, + kAvxBy256, + kAvxBy512, + }; + + // Returns name of given CopyMethod. + static std::string ToString(CopyMethod m); + + static const size_t kPageSize; + static const size_t kCacheLineSize; + + // Returns a random alignment offset in range [0..kPageSize-1]. + static size_t RandomAlignment(uint64_t seed); + + // Helper to make MSAN happy. NOP if memory sanitizer is not enabled. + static void InitializeMemoryForSanitizer(char* addr, size_t size); + + // Constructs MalignBuffer of specified capacity. + MalignBuffer(size_t capacity); + + // Constructs and initializes MalignBuffer with specified alignment and + // content. Useful for unit tests. + // REQUIRES: + // alignment_offset < kPageSize + MalignBuffer(size_t alignment_offset, absl::string_view s); + + ~MalignBuffer(); + + // Initializes buffer to specified alignment. + // REQUIRES: + // alignment_offset < kPageSize + // length <= this.capacity_. + void Initialize(size_t alignment_offset, size_t length); + + const char* data() const { return buffer_address_; } + char* data() { return buffer_address_; } + size_t size() const { return length_; } + + // REQUIRES length <= capacity_. + void resize(size_t length); + + // Compares 'this' to 'that' returning empty string if identical. + // If not identical, returns a syndrome, currently Hamming distance, + // corrupted subrange bounds, and the diffs. + std::string Syndrome(const MalignBuffer& that) const; + + // Validated data copy from source to 'this'. + // 'this' must be appropriately sized. + // Returns syndrome upon copy failure. + std::string CopyFrom(const MalignBuffer& that, CopyMethod m); + + // Unvalidated copy to 'this'. + void CopyFrom(absl::string_view src, CopyMethod m); + void CopyFrom(size_t pos, absl::string_view src, CopyMethod m); + + // Randomly flushes cache lines. + void RandomFlush(std::knuth_b* rng) const; + + // Conventional or rep;sto memset operation, according to 'use_rep_stos'. + void Memset(size_t offset, unsigned char v, size_t length, bool use_rep_stos); + + // Memsets buffer to 'hole.v', using rep;stos operation if + // 'use_rep_stos' set; + void PunchHole(const PunchedHole& hole, bool use_rep_stos); + + // Hints to the OS to release the buffer's memory. + void MadviseDontNeed() const; + + // Returns random PunchedHole within 'this'. + MalignBuffer::PunchedHole RandomPunchedHole(uint64_t seed) const; + + private: + static size_t RoundUpToPageSize(size_t k); + std::string CorruptionSyndrome(const MalignBuffer& that) const; + std::string CrackId(uint64_t) const; + + const size_t capacity_; + void* base_address_ = nullptr; + + size_t alignment_offset_ = 0; + size_t length_ = 0; // Usable length + char* buffer_address_ = nullptr; +}; + +} // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_MALIGN_BUFFER_H_ diff --git a/pattern_generator.cc b/pattern_generator.cc new file mode 100644 index 0000000..39f79f7 --- /dev/null +++ b/pattern_generator.cc @@ -0,0 +1,224 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "pattern_generator.h" + +#include + +#include +#include +#include + +#include "log.h" + +namespace cpu_check { +namespace { + +// So-called Logistic Map with parameter 4.0. +// Floating point approximation aside, if v is in the closed unit interval than +// ChaoticF1(v) is in the closed unit interval. +template +T ChaoticF1(T v) { + return 4.0 * v * (1.0 - v); +} + +// Reciprocal-like function valid over closed unit interval. +template +T Recip(T v) { + return 1.0 / (v + 0.1); +} + +// Inverse of Recip for v in closed unit interval. +template +T Unrecip(T v) { + return (1.0 / v) - 0.1; +} + +template +T ReciprocatedChaos(T v) { + return Recip(ChaoticF1(Unrecip(v))); +} + +std::vector ReadDict() { + // Dictionary search paths + static const char* dicts[] = { + "/usr/share/dict/words", + "words", + }; + std::vector words; + std::ifstream f; + + for (const auto& d : dicts) { + f.open(d, std::ifstream::in); + if (f.is_open()) break; + f.clear(); + } + + if (!f.is_open()) return words; + + LOG(DEBUG) << "Reading words."; + + std::string word; + while (!f.eof()) { + std::getline(f, word); + words.push_back(word); + } + f.close(); + LOG(DEBUG) << "Read " << words.size() << " words."; + std::sort(words.begin(), words.end(), + [](const std::string& a, const std::string& b) { + return a.size() < b.size(); + }); + return words; +} +} // namespace + +FloatingPointResults FillBufferSystematic::Generate( + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + const uint64_t pid = getpid(); + + // Format: 2 bytes of PID, 3 bytes of round number, 3 bytes of offset. + // Note: Perhaps should be AC-modulated. Perhaps should be absolute aligned + // for easier recognition. + // Note: appropriate for LE machines only. + FloatingPointResults fp; + fp.d = std::max(round, 2); + for (size_t i = 0; i * 8 < b->size(); i++) { + const size_t p = 8 * i; + const size_t k = std::min(8, b->size() - p); + const uint64_t v = + ((pid & 0xffff) << 48) | ((round & 0xffffff) << 24) | (i & 0xffffff); + for (size_t m = 0; m < k; m++) { + (b->data())[p + m] = *(reinterpret_cast(&v) + m); + } + if (exercise_floating_point) { + fp.d = ReciprocatedChaos(fp.d); + } + } + return fp; +} + +FloatingPointResults FillBufferRandom::Generate( + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + std::knuth_b rng(round); + std::uniform_int_distribution dist( + 0, std::numeric_limits::max()); + FloatingPointResults fp; + fp.f = std::max(round, 2); + size_t p = 0; + const size_t length = b->size(); + // Repeatedly append random number (one to eight) random bytes. + while (p < length) { + const size_t max_span = std::min(length - p, 8); + const size_t z = std::uniform_int_distribution(1, max_span)(rng); + const uint64_t v = dist(rng); + b->CopyFrom(p, absl::string_view(reinterpret_cast(&v), z), + copy_method); + p += z; + if (exercise_floating_point) { + fp.f = ReciprocatedChaos(fp.f); + } + } + return fp; +} + +FloatingPointResults FillBufferText::Generate( + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + std::knuth_b rng(round); + std::exponential_distribution dist(20); + FloatingPointResults fp; + fp.ld = std::max(round, 2); + const size_t bufsize = b->size(); + size_t pos = 0; + while (pos < bufsize) { + const size_t r = std::min(static_cast(dist(rng) * words_.size()), + words_.size() - 1); + const auto& word = words_[r]; + const size_t wordlen = word.size(); + if (pos + wordlen >= bufsize) { + break; + } + b->CopyFrom(pos, word, copy_method); + pos += wordlen; + if (pos < bufsize) { + b->Memset(pos, ' ', 1, use_repstos); + pos++; + } + if (exercise_floating_point) { + fp.ld = ReciprocatedChaos(fp.ld); + } + } + // Pad with spaces + b->Memset(pos, ' ', bufsize - pos, use_repstos); + return fp; +} + +FloatingPointResults FillBufferGrilledCheese::Generate( + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + std::knuth_b rng(round); + FloatingPointResults fp; + fp.f = std::max(round, 2); + fp.d = std::max(round, 2); + const size_t kAdvance = 15; + const size_t kWindow = 64; + unsigned char flavor = 0; + b->Memset(0, 0, b->size(), use_repstos); + for (int base = kWindow; base < b->size(); base += kAdvance) { + if (std::uniform_int_distribution(0, 1)(rng)) continue; + flavor++; + const size_t start = + std::uniform_int_distribution(base - kWindow, base)(rng); + const size_t end = std::uniform_int_distribution(start, base)(rng); + b->Memset(start, flavor, 1 + end - start, use_repstos); + if (exercise_floating_point) { + fp.f = ReciprocatedChaos(fp.f); + fp.d = ReciprocatedChaos(fp.d); + } + } + return fp; +} + +PatternGenerators::PatternGenerators() : words_(ReadDict()) { + if (words_.empty()) { + LOG(ERROR) << "No word list found."; + exit(1); + } + generators_.emplace_back(new FillBufferSystematic()); + generators_.emplace_back(new FillBufferRandom()); + generators_.emplace_back(new FillBufferText(words_)); + generators_.emplace_back(new FillBufferGrilledCheese()); +} + +const PatternGenerator& PatternGenerators::RandomGenerator( + uint64_t round) const { + std::knuth_b rng(round); + const size_t k = + std::uniform_int_distribution(0, generators_.size() - 1)(rng); + return *generators_[k]; +} + +FloatingPointResults PatternGenerators::Generate( + const PatternGenerator& generator, const MalignBuffer::PunchedHole& hole, + uint64_t round, MalignBuffer::CopyMethod copy_method, bool use_repstos, + bool exercise_floating_point, MalignBuffer* b) const { + const FloatingPointResults f = generator.Generate( + round, copy_method, use_repstos, exercise_floating_point, b); + b->PunchHole(hole, use_repstos); + return f; +} +} // namespace cpu_check diff --git a/pattern_generator.h b/pattern_generator.h new file mode 100644 index 0000000..db8b76d --- /dev/null +++ b/pattern_generator.h @@ -0,0 +1,123 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_PATTERN_GENERATOR_H_ +#define THIRD_PARTY_CPU_CHECK_PATTERN_GENERATOR_H_ + +#include +#include + +#include "malign_buffer.h" + +namespace cpu_check { + +class PatternGenerators; + +struct FloatingPointResults { + bool operator==(const FloatingPointResults& other) const { + return f == other.f && d == other.d && ld == other.ld; + } + bool operator!=(const FloatingPointResults& other) const { + return f != other.f || d != other.d || ld != other.ld; + } + + float f = 0.0; + double d = 0.0; + long double ld = 0.0; +}; + +class PatternGenerator { + public: + virtual ~PatternGenerator() {} + virtual std::string Name() const = 0; + + virtual FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, + bool exercise_floating_point, + MalignBuffer*) const = 0; +}; + +// Fills buffer with a systematic pattern. +// Returns iterate of chaotic floating point function of 'seed', with some +// reciprocal torture. +class FillBufferSystematic : public PatternGenerator { + public: + std::string Name() const override { return "Systematic"; } + FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer*) const override; +}; + +// Fills buffer with a random pattern. +// Returns iterate of chaotic floating point function of 'seed', with some +// reciprocal torture. +class FillBufferRandom : public PatternGenerator { + public: + std::string Name() const override { return "Random"; } + FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer*) const override; +}; + +// Fills buffer with a compressible pattern. +// Returns iterate of chaotic floating point function of 'seed', with some +// reciprocal torture. +class FillBufferText : public PatternGenerator { + public: + FillBufferText(const std::vector& words) : words_(words) {} + std::string Name() const override { return "Text"; } + FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer*) const override; + + private: + const std::vector& words_; +}; + +// memset (conventional or rep;stos) randomly aligned, random width, randomly +// overlapped stretches of buffer. Constants aim to hit multiple times in +// cache lines and buffers. Untuned and based on nothing but hunches. +class FillBufferGrilledCheese : public PatternGenerator { + public: + std::string Name() const override { return "Cheese"; } + FloatingPointResults Generate(uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer*) const override; +}; + +class PatternGenerators { + public: + PatternGenerators(); + const PatternGenerator& RandomGenerator(uint64_t round) const; + + FloatingPointResults Generate(const PatternGenerator& generator, + const MalignBuffer::PunchedHole& hole, + uint64_t round, + MalignBuffer::CopyMethod copy_method, + bool use_repstos, bool exercise_floating_point, + MalignBuffer* b) const; + + const std::vector& words() const { return words_; } + + private: + const std::vector words_; + std::vector> generators_; +}; +} // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_PATTERN_GENERATOR_H_ diff --git a/silkscreen.cc b/silkscreen.cc new file mode 100644 index 0000000..d18fcc2 --- /dev/null +++ b/silkscreen.cc @@ -0,0 +1,91 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "silkscreen.h" + +#include + +#include + +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "utils.h" + +namespace cpu_check { +static const size_t kPageSize = sysconf(_SC_PAGESIZE); + +Silkscreen::Silkscreen(const std::vector &tid_list) + : buffer_address_(static_cast(aligned_alloc( + kPageSize, kPageSize * ((kSize + kPageSize - 1) / kPageSize)))) { + std::knuth_b rng; + std::uniform_int_distribution dist(0, tid_list.size() - 1); + for (size_t k = 0; k < kSize; k++) { + size_t w = dist(rng); + const int o = tid_list[w]; + slot_count_[o]++; + owner_.push_back(o); + } +} + +absl::Status Silkscreen::WriteMySlots(int tid, uint64_t round) { + uint64_t j = 0; + for (size_t k = 0; k < kSize; k++) { + if (owner(k) == tid) { + *data(k) = static_cast(round); + j++; + } + } + if (j != slot_count_[tid]) { + std::string err = absl::StrCat(Json("written", j), ", ", + Json("expected", slot_count_[tid])); + return absl::Status(absl::StatusCode::kInternal, err); + } + return absl::OkStatus(); +} + +// When Silkscreen fails, it often fails, on several bad machines, +// in a surprising way: all slots owned by reading tid are +// are corrupt, in a way that suggests the previous round's +// writes never happened. Weird, and deserves some study, but +// meanwhile the log spew is suppressed by reporting only the last +// error and the error count. +absl::Status Silkscreen::CheckMySlots(int tid, uint64_t round) const { + const char expected = static_cast(round); + uint64_t slots_read = 0; + uint64_t error_count = 0; + std::string last_error; + + for (size_t k = 0; k < Silkscreen::kSize; k++) { + if (owner(k) != tid) continue; + slots_read++; + const char v = *data(k); + if (v == expected) continue; + error_count++; + last_error = absl::StrCat(Json("position", k), ", ", Json("is", v), ", ", + Json("expected", expected)); + } + if (slot_count(tid) != slots_read) { + last_error = absl::StrCat(Json("read", slots_read), ", ", + Json("expected", slot_count(tid))); + error_count++; + } + if (error_count > 0) { + return absl::Status( + absl::StatusCode::kInternal, + absl::StrCat(last_error, ", ", Json("errors", error_count))); + } else { + return absl::OkStatus(); + } +} +} // namespace cpu_check diff --git a/silkscreen.h b/silkscreen.h new file mode 100644 index 0000000..9a13590 --- /dev/null +++ b/silkscreen.h @@ -0,0 +1,63 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include "absl/status/status.h" + +#ifndef THIRD_PARTY_CPU_CHECK_SILKSCREEN_H_ +#define THIRD_PARTY_CPU_CHECK_SILKSCREEN_H_ + +namespace cpu_check { +// Rudimentary coherence/uncore tester. +// Randomly assigns each slot of a seemingly shared buffer to a single tid, +// creating only "false sharing". +// Thus each slot, regardless of alignment, must obey program order unless the +// machine is broken. +// To be toughened, e.g.: +// Widen the slots a bit +// Control the sharing more tightly, e.g. each cache line split between 2 tids +// Maybe checksum the indices to distinguish core-local compute errors from +// coherence errors, but that's perhaps easier said than done effectively. +// As it stands, it may be particularly hard to localize failures. Though that's +// always going to be a bit hard, which is the point. One technique might be +// to leave this alone and to run on subsets of cores and sockets. +class Silkscreen { + public: + static constexpr size_t kSize = 1000 * 1000; // Size of buffer + + Silkscreen(const std::vector& tid_list); + ~Silkscreen() { free(buffer_address_); } + + // Writes value derived from 'round' into all slots owned by 'tid'. + // Returns non-OK Status with JSON-formatted message upon error. + absl::Status WriteMySlots(int tid, uint64_t round); + + // Checks all slots owned by 'tid' for value appropriate to 'round'. + // Returns non-OK Status with JSON-formatted message upon error. + absl::Status CheckMySlots(int tid, uint64_t round) const; + + private: + int owner(size_t k) const { return owner_[k]; } + size_t size() const { return owner_.size(); } + int slot_count(int owner) const { return slot_count_.at(owner); } + const char* data(size_t k) const { return buffer_address_ + k; } + char* data(size_t k) { return buffer_address_ + k; } + + std::vector owner_; // const after initialization + std::map slot_count_; // const after initialization + char* const buffer_address_; +}; +} // namespace cpu_check +#endif // THIRD_PARTY_CPU_CHECK_SILKSCREEN_H_ diff --git a/stopper.h b/stopper.h new file mode 100644 index 0000000..8f80ec9 --- /dev/null +++ b/stopper.h @@ -0,0 +1,58 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_CPU_CHECK_STOPPER_H_ +#define THIRD_PARTY_CPU_CHECK_STOPPER_H_ + +#include + +#include +#include + +#include "utils.h" + +class Stopper { + public: + // Infinite timeout if 'timeout' <= 0. + Stopper(int timeout) + : t_stop_(timeout <= 0 ? std::numeric_limits::infinity() + : TimeInSeconds() + timeout) {} + + // Returns true if time has expired or Stop has been invoked. + // Thread safe. + bool Expired() const { return stopped_ || TimeInSeconds() > t_stop_; } + + // Sleeps for the minimum of 't' and remaining run time. + // Thread safe. + void BoundedSleep(int t) const { + if (std::isinf(t_stop_)) { + sleep(t); + } else { + const double remaining = t_stop_ - TimeInSeconds(); + if (!stopped_ && remaining > 0) { + sleep(std::min(t, ceil(remaining))); + } + } + } + + // Causes timeout to expire now. + // Thread safe. + void Stop() { stopped_ = true; } + + private: + const double t_stop_; + std::atomic_bool stopped_ = false; +}; + +#endif // THIRD_PARTY_CPU_CHECK_STOPPER_H_ diff --git a/utils.cc b/utils.cc index f8e4193..69b606f 100644 --- a/utils.cc +++ b/utils.cc @@ -18,6 +18,7 @@ #include #include "log.h" +#include "absl/strings/str_cat.h" static const std::string host_name = []() { char host[256]; @@ -34,6 +35,21 @@ double TimeInSeconds() { return ((tv.tv_sec * 1e6) + tv.tv_usec) / 1e6; } +std::string HexData(const char* s, uint32_t l) { + const char d[16] = {'0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; + std::string o; + o.resize(l << 1); + for (uint32_t i = 0; i < l; i++) { + uint8_t b = s[i]; + o[(i << 1)] = d[(b >> 4) & 0xf]; + o[(i << 1) + 1] = d[b & 0xf]; + } + return o; +} + +std::string HexStr(const std::string& s) { return HexData(s.data(), s.size()); } + std::string Json(const std::string& field, int v) { return "\"" + field + "\": " + std::to_string(v); } @@ -50,12 +66,12 @@ std::string JsonBool(const std::string& field, bool v) { return "\"" + field + "\": " + (v ? "true" : "false"); } -std::string Json(const std::string& field, const std::string& v) { - return "\"" + field + "\": \"" + v + "\""; +std::string Json(const std::string& field, absl::string_view v) { + return absl::StrCat("\"", field, "\": \"", v, "\""); } -std::string JsonRecord(const std::string& name, const std::string& v) { - return "\"" + name + "\": { " + v + " }"; +std::string JsonRecord(const std::string& name, absl::string_view v) { + return absl::StrCat("\"", name, "\": { ", v, " }"); } // Emits null field. diff --git a/utils.h b/utils.h index 53f8f2c..954d13e 100644 --- a/utils.h +++ b/utils.h @@ -17,15 +17,19 @@ #include +#include "absl/strings/string_view.h" double TimeInSeconds(); +std::string HexData(const char* s, uint32_t l); +std::string HexStr(const std::string& s); + std::string Json(const std::string& field, int v); std::string Json(const std::string& field, uint64_t v); std::string Json(const std::string& field, double v); std::string JsonBool(const std::string& field, bool v); -std::string Json(const std::string& field, const std::string& v); -std::string JsonRecord(const std::string& name, const std::string& v); +std::string Json(const std::string& field, absl::string_view v); +std::string JsonRecord(const std::string& name, absl::string_view v); // Emits null field. std::string JsonNull(const std::string& field);