eric/cpu-check
1
0
Fork 0
Code Pull Requests Actions Packages Releases Activity
Files
fa4b8965d7e78a539cd20fa0f524ff4d930d3ada
cpu-check/cpu_check.cc
Kevin Boyd fa4b8965d7 Initial commit
2020-05-08 13:33:05 -07:00

2076 lines
64 KiB
C++
Raw Blame History

// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#undef NDEBUG
#include "config.h"
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <errno.h>
#include <fcntl.h>
#if defined(__x86_64__) || defined(__i386__)
#include <immintrin.h>
#endif
#include <sched.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
#ifdef IN_GOOGLE3
#include "third_party/zlib/zlib.h"
#else
#include <zlib.h>
#endif
#include <algorithm>
#include <atomic>
#include <cassert>
#include <climits>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <ctime>
#include <fstream>
#include <functional>
#include <iomanip>
#include <iostream>
#include <limits>
#include <list>
#include <memory>
#include <mutex>
#include <random>
#include <sstream>
#include <string>
#include <thread>
#include <utility>
#include <vector>
#ifdef IN_GOOGLE3
#include "third_party/openssl/crypto.h"
#include "third_party/openssl/evp.h"
#else
#include <openssl/crypto.h>
#include <openssl/evp.h>
#endif
#ifdef IN_GOOGLE3
#include "third_party/absl/debugging/failure_signal_handler.h"
#include "third_party/absl/debugging/symbolize.h"
#endif
#include "crc32c.h"
#include "third_party/farmhash/src/farmhash.h"
#include "fvt_controller.h"
#include "log.h"
#include "utils.h"
#if defined(__i386__) || defined(__x86_64__)
#define X86_TARGET_ATTRIBUTE(s) __attribute__ ((target (s)))
#else
#define X86_TARGET_ATTRIBUTE(s)
#endif
#undef HAS_FEATURE_MEMORY_SANITIZER
#if defined(__has_feature)
# if __has_feature(memory_sanitizer)
#define HAS_FEATURE_MEMORY_SANITIZER
# endif
#endif
// Helper to make MSAN happy. NOP if memory sanitizer is not enabled.
void InitializeMemoryForSanitizer(char* addr, size_t size) {
#ifdef HAS_FEATURE_MEMORY_SANITIZER
std::default_random_engine rnd;
std::uniform_int_distribution<int> dist(std::numeric_limits<char>::min(),
std::numeric_limits<char>::max());
for (size_t i = 0; i < size; i++) {
addr[i] = dist(rnd);
}
#endif
}
inline void __movsb(char *dst, const char *src, size_t size) {
#if defined(__i386__) || defined(__x86_64__)
__asm__ __volatile__("rep movsb"
: "+D"(dst), "+S"(src), "+c"(size)
:
: "memory");
#else
LOG(FATAL) << "Cannot rep;movsb";
#endif
}
inline void __stosb(void *dst, unsigned char c, size_t size) {
#if defined(__i386__) || defined(__x86_64__)
__asm__ __volatile__("rep stosb"
: "+D"(dst), "+c"(size)
: "a"(c)
: "memory");
#else
LOG(FATAL) << "Cannot rep;stosb";
#endif
}
inline void __sse_128_memcpy(char *dst, const char *src, size_t size) {
#if (defined(__i386__) || defined(__x86_64__))
size_t blks = size / 16;
for (int i = 0; i < blks; i++) {
_mm_storeu_si128(
reinterpret_cast<__m128i *>(dst) + i,
_mm_loadu_si128(reinterpret_cast<const __m128i *>(src) + i));
}
memcpy(dst + blks * 16, src + blks * 16, size - blks * 16);
#else
LOG(FATAL) << "SSE not available";
#endif
}
X86_TARGET_ATTRIBUTE("avx")
inline void __avx_256_memcpy(char *dst, const char *src, size_t size) {
#if (defined(__i386__) || defined(__x86_64__))
size_t blks = size / 32;
for (int i = 0; i < blks; i++) {
_mm256_storeu_si256(
reinterpret_cast<__m256i *>(dst) + i,
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(src) + i));
}
memcpy(dst + blks * 32, src + blks * 32, size - blks * 32);
#else
LOG(FATAL) << "x86 only";
#endif
}
X86_TARGET_ATTRIBUTE("avx512f")
inline void __avx_512_memcpy(char *dst, const char *src, size_t size) {
#if (defined(__i386__) || defined(__x86_64__))
size_t blks = size / 64;
for (int i = 0; i < blks; i++) {
_mm512_storeu_si512(
reinterpret_cast<__m512i *>(dst) + i,
_mm512_loadu_si512(reinterpret_cast<const __m512i *>(src) + i));
}
memcpy(dst + blks * 64, src + blks * 64, size - blks * 64);
#else
LOG(FATAL) << "x86 only";
#endif
}
static const double t0 = TimeInSeconds();
static void SleepForMillis(int64_t millis) {
// LOG(INFO) << "sleeping " << millis;
int64_t micros = 1000 * millis;
while (micros > 0) {
int mm = std::min<int>(1000000, micros);
int rc = usleep(mm);
if (rc) {
LOG(ERROR) << "cant sleep";
}
micros -= mm;
}
}
static long pagesize = sysconf(_SC_PAGESIZE);
static long cache_line_size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
std::string HexData(const char *s, uint32_t l) {
const char d[16] = {'0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
std::string o;
o.resize(l << 1);
for (uint32_t i = 0; i < l; i++) {
uint8_t b = s[i];
o[(i << 1)] = d[(b >> 4) & 0xf];
o[(i << 1) + 1] = d[b & 0xf];
}
return o;
}
std::string HexStr(const std::string &s) { return HexData(s.data(), s.size()); }
std::atomic_bool exiting(false);
std::atomic_uintmax_t errorCount(0);
std::atomic_uintmax_t successCount(0);
static constexpr uintmax_t kErrorLimit = 2000;
#if defined(__i386__) || defined(__x86_64__)
const bool is_x86 = true;
#else
const bool is_x86 = false;
#endif
// So-called Logistic Map with parameter 4.0.
// Floating point approximation aside, if 0 < v < 1 then 0 < ChaoticF1(v) < 1.
static inline double ChaoticF1(double v) {
return 4.0 * v * (1.0 - v);
}
#if defined(__i386__) || defined(__x86_64__)
static bool can_do_avx() {
__builtin_cpu_init();
return __builtin_cpu_supports("avx");
}
static bool can_do_avx512f() {
__builtin_cpu_init();
return __builtin_cpu_supports("avx512f");
}
static bool can_do_fma() {
__builtin_cpu_init();
return __builtin_cpu_supports("fma");
}
static bool can_do_fvt() {
return geteuid() == 0; // need write access to MSRs.
}
#else
static bool can_do_avx() { return false; }
static bool can_do_avx512f() { return false; }
static bool can_do_fma() { return false; }
static bool can_do_fvt() { return false; } // x86-only for now.
#endif
bool do_madvise = true;
bool do_repmovsb = is_x86;
bool do_sse_128_memcpy = is_x86;
bool do_avx_256_memcpy = can_do_avx();
bool do_avx_512_memcpy = can_do_avx512f();
bool do_avx_heavy = can_do_avx();
bool do_compress = true;
bool do_encrypt = true;
bool do_hashes = true;
bool do_misalign = true;
bool do_hop = true;
bool do_ssl_self_check = true;
bool do_flush = false; // Default: disabled for now
bool do_provenance = false;
bool do_repstosb = is_x86;
bool do_freq_sweep = false;
bool do_freq_hi_lo = false;
bool do_noise = false;
bool do_invert_cores = false;
int fixed_min_frequency = 0;
int fixed_max_frequency = 0;
bool do_fvt = can_do_fvt();
bool do_fast_string_ops = true;
int seconds_per_freq = 300;
uintmax_t error_limit = kErrorLimit;
bool SetAffinity(int id) {
int err = 0;
#ifdef __linux__
cpu_set_t cset;
CPU_ZERO(&cset);
CPU_SET(id, &cset);
err = sched_setaffinity(0, sizeof(cset), &cset);
std::atomic_thread_fence(std::memory_order_seq_cst);
if (err) {
err = errno;
}
#elif defined(__NetBSD__)
cpuset_t *cset;
cset = cpuset_create();
if (cset == nullptr) {
LOG(ERROR) << "cpuset_create failed: " << strerror(errno);
return false;
}
cpuset_set(id, cset);
err = pthread_setaffinity_np(pthread_self(), cpuset_size(cset), cset);
std::atomic_thread_fence(std::memory_order_seq_cst);
cpuset_destroy(cset);
#endif
if (err != 0) {
LOG_EVERY_N_SECS(WARN, 30)
<< "setaffinity to tid: " << id << " failed: " << strerror(err);
}
return err == 0;
}
std::vector<std::string> ReadDict() {
// Dictionary search paths
static const char *dicts[] = {
"/usr/share/dict/words",
"words",
};
std::vector<std::string> words;
std::ifstream f;
for (const auto &d : dicts) {
f.open(d, std::ifstream::in);
if (f.is_open()) break;
f.clear();
}
if (!f.is_open()) return words;
LOG(DEBUG) << "Reading words.";
std::string word;
while (!f.eof()) {
std::getline(f, word);
words.push_back(word);
}
f.close();
LOG(DEBUG) << "Read " << words.size() << " words.";
std::sort(words.begin(), words.end(),
[](const std::string &a, const std::string &b) {
return a.size() < b.size();
});
return words;
}
class MalignBuffer {
public:
struct PunchedHole {
std::string ToString() const;
size_t start = 0;
size_t length = 0;
unsigned char v = 0x53;
};
enum CopyMethod {
kMemcpy,
kRepMov,
kSseBy128,
kAvxBy256,
kAvxBy512,
};
static std::string ToString(CopyMethod m) {
switch (m) {
case kMemcpy:
return "memcpy";
case kRepMov:
return "rep;mov";
case kSseBy128:
return "sse:128";
case kAvxBy256:
return "avx:256";
case kAvxBy512:
return "avx:512";
}
}
// Provides buffer with specified alignment.
MalignBuffer(size_t capacity)
: capacity_(capacity),
base_address_(aligned_alloc(pagesize, capacity + pagesize)) {
assert(base_address_);
// There are lots of places that use unitialized MalignBuffer. So just
// fill some pseudo-random bytes if cpu_check is compiled with msan.
InitializeMemoryForSanitizer(static_cast<char*>(base_address_), capacity_);
}
~MalignBuffer() { free(base_address_); }
// REQUIRES: alignment_offset + length <= capacity_.
void Initialize(size_t alignment_offset, size_t length);
const char* data() const { return buffer_address_; }
char* data() { return buffer_address_; }
size_t size() const { return length_; }
// REQUIRES alignment_offset + length <= capacity_.
void resize(size_t length);
// Compares 'this' to 'that' returning empty string if identical.
// If not identical, returns a syndrome, currently Hamming distance,
// corrupted subrange bounds, and the diffs.
std::string Syndrome(const MalignBuffer& that) const;
// Validated data copy from source to 'this'.
// 'this' must be appropriately sized.
// Returns syndrome upon copy failure.
std::string CopyFrom(const MalignBuffer& that, CopyMethod m);
// Unvalidated copy to 'this'.
void CopyFrom(const char* src, size_t length, CopyMethod m);
void CopyFrom(size_t pos, const char* src, size_t length, CopyMethod m);
// Randomly flushes cache lines.
void RandomFlush(std::knuth_b* rng) const;
// Conventional or rep;sto memset operation, according to 'use_rep_stos'.
void Memset(size_t offset, unsigned char v, size_t length, bool use_rep_stos);
// Memsets buffer to 'hole.v', using rep;stos operation if
// 'use_rep_stos' set;
void PunchHole(const PunchedHole& hole, bool use_rep_stos);
private:
std::string CorruptionSyndrome(const MalignBuffer& that) const;
std::string CrackId(uint64_t) const;
const size_t capacity_;
void* base_address_ = nullptr;
size_t alignment_offset_ = 0;
size_t length_ = 0; // Usable length
char* buffer_address_ = nullptr;
};
void MalignBuffer::Initialize(size_t alignment_offset, size_t length) {
assert(alignment_offset + length <= capacity_);
alignment_offset_ = alignment_offset;
length_ = length;
buffer_address_ = static_cast<char*>(base_address_) + alignment_offset_;
}
void MalignBuffer::resize(size_t length) {
Initialize(alignment_offset_, length);
}
std::string MalignBuffer::CopyFrom(const MalignBuffer& that, CopyMethod m) {
CopyFrom(that.data(), that.size(), m);
return Syndrome(that);
}
void MalignBuffer::CopyFrom(const char* src, size_t length, CopyMethod m) {
assert(size() == length);
CopyFrom(0, src, length, m);
}
void MalignBuffer::CopyFrom(size_t pos, const char* src, size_t length,
CopyMethod m) {
assert(pos + length <= size());
switch (m) {
case kMemcpy:
// Assumes memcpy doesn't use rep;movsb; false in lots of environments.
memcpy(data() + pos, src, length);
break;
case kRepMov:
__movsb(data() + pos, src, length);
break;
case kSseBy128:
__sse_128_memcpy(data() + pos, src, length);
break;
case kAvxBy256:
__avx_256_memcpy(data() + pos, src, length);
break;
case kAvxBy512:
__avx_512_memcpy(data() + pos, src, length);
break;
}
}
std::string MalignBuffer::Syndrome(const MalignBuffer& that) const {
std::stringstream s;
std::string syndrome = CorruptionSyndrome(that);
if (syndrome.empty()) return "";
s << syndrome << ", \"this\": \"" << static_cast<const void *>(data())
<< "\", "
<< "\"that\": \"" << static_cast<const void *>(that.data()) << "\"";
return s.str();
}
std::string MalignBuffer::CorruptionSyndrome(const MalignBuffer& that) const {
std::stringstream s;
if (size() != that.size()) {
s << Json("unequalSizeThis", size()) << ", "
<< Json("unequalSizeThat", that.size());
return s.str();
}
bool failed_memcmp = memcmp(data(), that.data(), that.size());
int wrong_bytes = 0;
int wrong_bits = 0;
int byte_faults = 0;
int first_wrong = INT_MAX;
int last_wrong = INT_MIN;
std::vector<int> lane_errors(8, 0);
for (size_t i = 0; i < size(); i++) {
unsigned char a = *(data() + i);
unsigned char b = *(that.data() + i);
unsigned char d = a ^ b;
if (d) {
first_wrong = std::min<int>(first_wrong, i);
last_wrong = std::max<int>(last_wrong, i);
byte_faults |= d;
wrong_bytes++;
wrong_bits += __builtin_popcount(d);
for (size_t i = 0; i < 8; i++) {
if ((d >> i) & 1) {
lane_errors[i]++;
}
}
}
}
if (wrong_bits || wrong_bytes) {
const int range_width = (last_wrong - first_wrong) + 1;
s << Json("cmpResult",
(failed_memcmp ? "Failed_Memcmp" : "**Passed_Memcmp**"))
<< ", " << Json("wrongByteCount", wrong_bytes) << ", "
<< Json("wrongBitCount", wrong_bits) << ", "
<< Json("corruptionWidth", range_width) << ", "
<< Json("corruptStart", first_wrong) << ", "
<< Json("corruptByteBitMask", byte_faults) << ", "
<< "\"byBitLane\": [";
for (size_t i = 0; i < 8; i++) {
if (i) s << ", ";
s << lane_errors[i];
}
s << " ] ";
// Dump up to 64 corrupted locations.
std::stringstream dump;
dump << " \"byteErrors\": [ " << std::hex;
uint64_t buf_a = 0;
uint64_t buf_b = 0;
for (size_t k = 0; k < std::min(64, range_width); k++) {
uint8_t a = *(data() + first_wrong + k);
uint8_t b = *(that.data() + first_wrong + k);
if (k) dump << ", ";
dump << "[ " << std::setw(2) << "\"0x" << static_cast<int>(a) << "\", "
<< std::setw(2) << "\"0x" << static_cast<int>(b) << "\" ";
buf_a = (buf_a >> 8) | static_cast<uint64_t>(a) << 56;
buf_b = (buf_b >> 8) | static_cast<uint64_t>(b) << 56;
if ((k >= 7) && (7 == ((first_wrong + k) % 8))) {
dump << ", " << CrackId(buf_a) << ", " << CrackId(buf_b);
buf_a = 0;
buf_b = 0;
}
dump << " ]";
}
dump << " ] ";
return s.str() + ", " + dump.str();
} else {
if (!failed_memcmp) return "";
return Json("cmpResult", "**Failed_Memcmp**");
}
}
std::string MalignBuffer::CrackId(uint64_t v) const {
std::stringstream s;
s << std::hex << " [\"0x" << std::setw(4) << (v >> 48) << "\", \"0x"
<< std::setw(6) << ((v >> 24) & 0xffffff) << "\", \"0x" << std::setw(6)
<< (v & 0xffffff) << "\"]";
return s.str();
}
void MalignBuffer::RandomFlush(std::knuth_b* rng) const {
#if defined(__i386__) || defined(__x86_64__)
// Note: no barriers used.
const char* p = buffer_address_ + alignment_offset_;
while (p < buffer_address_ + length_) {
if (std::uniform_int_distribution<int>(0, 1)(*rng)) {
__builtin_ia32_clflush(p);
}
p += cache_line_size;
}
#endif
}
std::string MalignBuffer::PunchedHole::ToString() const {
if (length) {
return JsonRecord("hole",
Json("start", start) + ", " + Json("length", length)
+ ", " + Json("v", static_cast<int>(v)));
} else {
return JsonNull("hole");
}
}
void MalignBuffer::Memset(size_t offset, unsigned char v, size_t length,
bool use_rep_stos) {
if (use_rep_stos) {
__stosb(data() + offset, v, length);
} else {
memset(data() + offset, v, length);
}
}
void MalignBuffer::PunchHole(const PunchedHole& hole, bool use_rep_stos) {
if (hole.length) {
Memset(hole.start, hole.v, hole.length, use_rep_stos);
}
}
// x86 AVX usage has complicated core power effects. This code tries
// to provoke some power transitions that don't otherwise happen.
// While it's at it, it lightly checks results, but that's not the central
// goal. ToDo: maybe toughen the correctness checking.
//
// The power policies are governed by a number of opaque parameters; this code
// is based on a lot of guesses.
class Avx {
public:
Avx() {}
// Activate AVX depending on throw of the dice.
// Returns syndrome if computational error detected.
std::string MaybeGoHot();
// Does a bit of computing if in a "hot" mode.
// Returns syndrome if computational error detected.
std::string BurnIfAvxHeavy();
private:
constexpr static int kIterations = 5000;
std::string Avx256(int rounds);
std::string Avx256FMA(int rounds);
std::string Avx512(int rounds);
int level_ = 0;
std::knuth_b rng_;
};
std::string Avx::MaybeGoHot() {
if (std::uniform_int_distribution<int>(0, 1)(rng_)) {
// Don't provoke.
level_ = 0;
return "";
}
if (can_do_avx512f()) {
// Processor supports both AVX and AVX512.
level_ = std::uniform_int_distribution<int>(0, 1)(rng_) ? 3 : 5;
} else {
// Processor supports only AVX.
level_ = 3;
}
return BurnIfAvxHeavy();
}
std::string Avx::BurnIfAvxHeavy() {
if (level_ == 3) {
return can_do_fma() ? Avx256FMA(kIterations) : Avx256(kIterations);
}
if (level_ == 5) {
return Avx512(kIterations);
}
return "";
}
// See notes for Avx512 below
X86_TARGET_ATTRIBUTE("avx")
std::string Avx::Avx256(int rounds) {
#if (defined(__i386__) || defined(__x86_64__))
const __m256d minus_four = _mm256_set1_pd(-4.0);
__m256d x[4];
for (int k = 0; k < 4; k++) {
x[k] =
_mm256_set1_pd(std::uniform_real_distribution<double>(0.0, 1.0)(rng_));
}
double *gross_x[4] = {
reinterpret_cast<double *>(&x[0]),
reinterpret_cast<double *>(&x[1]),
reinterpret_cast<double *>(&x[2]),
reinterpret_cast<double *>(&x[3]),
};
for (int i = 0; i < rounds; i++) {
__m256d a[4];
a[0] = _mm256_sub_pd(_mm256_mul_pd(x[0], x[0]), x[0]);
a[1] = _mm256_sub_pd(_mm256_mul_pd(x[1], x[1]), x[1]);
a[2] = _mm256_sub_pd(_mm256_mul_pd(x[2], x[2]), x[2]);
a[3] = _mm256_sub_pd(_mm256_mul_pd(x[3], x[3]), x[3]);
x[0] = _mm256_mul_pd(minus_four, a[0]);
x[1] = _mm256_mul_pd(minus_four, a[1]);
x[2] = _mm256_mul_pd(minus_four, a[2]);
x[3] = _mm256_mul_pd(minus_four, a[3]);
}
for (int k = 1; k < 4; k++) {
for (int i = 0; i < 4; i++) {
if (gross_x[k][i] != gross_x[k][0]) {
return "avx256 pd";
}
}
}
#endif
return "";
}
// See notes for Avx512 below
X86_TARGET_ATTRIBUTE("avx,fma")
std::string Avx::Avx256FMA(int rounds) {
#if (defined(__i386__) || defined(__x86_64__))
const __m256d minus_four = _mm256_set1_pd(-4.0);
__m256d x[4];
for (int k = 0; k < 4; k++) {
x[k] =
_mm256_set1_pd(std::uniform_real_distribution<double>(0.0, 1.0)(rng_));
}
double *gross_x[4] = {
reinterpret_cast<double *>(&x[0]),
reinterpret_cast<double *>(&x[1]),
reinterpret_cast<double *>(&x[2]),
reinterpret_cast<double *>(&x[3]),
};
for (int i = 0; i < rounds; i++) {
__m256d a[4];
a[0] = _mm256_fmsub_pd(x[0], x[0], x[0]);
a[1] = _mm256_fmsub_pd(x[1], x[1], x[1]);
a[2] = _mm256_fmsub_pd(x[2], x[2], x[2]);
a[3] = _mm256_fmsub_pd(x[3], x[3], x[3]);
x[0] = _mm256_mul_pd(minus_four, a[0]);
x[1] = _mm256_mul_pd(minus_four, a[1]);
x[2] = _mm256_mul_pd(minus_four, a[2]);
x[3] = _mm256_mul_pd(minus_four, a[3]);
}
for (int k = 1; k < 4; k++) {
for (int i = 0; i < 4; i++) {
if (gross_x[k][i] != gross_x[k][0]) {
return "avx256 pd";
}
}
}
#endif
return "";
}
// Interleave AVX512 parallel calculation of iterates of f(x) = 4x(1-x).
// Hope compiler too dumb to see through this.
X86_TARGET_ATTRIBUTE("avx512f")
std::string Avx::Avx512(int rounds) {
#if (defined(__i386__) || defined(__x86_64__))
const __m512d minus_four = _mm512_set1_pd(-4.0);
__m512d x[4];
for (int k = 0; k < 4; k++) {
x[k] =
_mm512_set1_pd(std::uniform_real_distribution<double>(0.0, 1.0)(rng_));
}
double *gross_x[4] = {
reinterpret_cast<double *>(&x[0]),
reinterpret_cast<double *>(&x[1]),
reinterpret_cast<double *>(&x[2]),
reinterpret_cast<double *>(&x[3]),
};
for (int i = 0; i < rounds; i++) {
__m512d a[4];
a[0] = _mm512_fmsub_pd(x[0], x[0], x[0]);
a[1] = _mm512_fmsub_pd(x[1], x[1], x[1]);
a[2] = _mm512_fmsub_pd(x[2], x[2], x[2]);
a[3] = _mm512_fmsub_pd(x[3], x[3], x[3]);
x[0] = _mm512_mul_pd(minus_four, a[0]);
x[1] = _mm512_mul_pd(minus_four, a[1]);
x[2] = _mm512_mul_pd(minus_four, a[2]);
x[3] = _mm512_mul_pd(minus_four, a[3]);
}
for (int k = 1; k < 4; k++) {
for (int i = 0; i < 7; i++) {
if (gross_x[k][i] != gross_x[k][0]) {
return "avx512 pd";
}
}
}
#endif
return "";
}
// Produces noise of all kinds by running intermittently.
// There's a coarse cycle with four fine phases:
// Phase 0: Off
// Phase 1: High-speed on-off
// Phase 2: On
// Phase 3: High-speed on-off
class NoiseScheduler {
public:
// The following constants are just plain made up outta whole cloth.
// You could consider various power regulation time constants and thermal
// intertia and so forth. Or just make it up.
static constexpr int kCoarseMillis = 5000; // Coarse period in millis
static constexpr int kFineMillis = 50; // Fine period in millis
// Blocks until next scheduled activity
static void BlockUntilOn() {
bool was_blocked = false;
while (true) {
int64_t t = 1e3 * TimeInSeconds();
int64_t coarse_block = t / kCoarseMillis;
int64_t phase = coarse_block % 4;
if (phase == 2) {
if (was_blocked) {
// LOG(INFO) << "Coarse grained unblock";
}
was_blocked = false;
return; // On
}
if (phase == 0) {
// Wait til next phase and then re-evaluate.
SleepForMillis(((coarse_block + 1) * kCoarseMillis) - t);
was_blocked = true;
continue;
}
// Fine phase.
int64_t fine_block = t / kFineMillis;
if (fine_block % 2) {
if (was_blocked) {
// LOG(INFO) << "Fine grained unblock";
}
was_blocked = false;
return; // Fine-grained on
}
// Wait til next fine block and then re-evaluate.
SleepForMillis(((fine_block + 1) * kFineMillis) - t);
was_blocked = true;
}
}
};
// Rudimentary coherence/uncore tester.
// Randomly assigns each slot of a seemingly shared buffer to a single tid,
// creating only "false sharing".
// Thus each slot, regardless of alignment, must obey program order unless the
// machine is broken.
// To be toughened, e.g.:
// Widen the slots a bit
// Control the sharing more tightly, e.g. each cache line split between 2 tids
// Maybe checksum the indices to distinguish core-local compute errors from
// coherence errors, but that's perhaps easier said than done effectively.
// As it stands, it may be particularly hard to localize failures. Though that's
// always going to be a bit hard, which is the point. One technique might be
// to leave this alone and to run on subsets of cores and sockets.
class Silkscreen {
public:
static constexpr size_t kSize = 1000 * 1000; // Size of buffer
Silkscreen(const std::vector<int> &tid_list);
~Silkscreen() { free(buffer_address_); }
// Writes value derived from 'round' into all slots owned by 'tid'.
// Returns number of slots written.
uint64_t WriteMySlots(int tid, uint64_t round);
// Returns JSON-formatted error string if slot 'k' belongs to 'tid' and has
// value not properly corresponding with 'round'.
// Returns "=" if slot 'k' belongs to 'tid' and has expected value.
// Otherwise, if slot 'k' does not belong to 'tid', returns empty string.
std::string CheckMySlot(int tid, uint64_t round, size_t k) const;
private:
int owner(size_t k) const { return owner_[k]; }
size_t size() const { return owner_.size(); }
const char* data(size_t k) const { return buffer_address_ + k; }
char* data(size_t k) { return buffer_address_ + k; }
std::vector<uint16_t> owner_; // const after initialization
char* buffer_address_ = nullptr;
};
Silkscreen::Silkscreen(const std::vector<int> &tid_list)
: buffer_address_(static_cast<char*>(aligned_alloc(
pagesize,
pagesize * ((kSize + pagesize - 1) / pagesize) ))) {
std::knuth_b rng;
std::uniform_int_distribution<size_t> dist(0, tid_list.size() - 1);
for (size_t k = 0; k < kSize; k++) {
size_t w = dist(rng);
owner_.push_back(tid_list[w]);
}
}
uint64_t Silkscreen::WriteMySlots(int tid, uint64_t round) {
uint64_t j = 0;
for (size_t k = 0; k < kSize; k++) {
if (owner(k) == tid) {
*data(k) = static_cast<char>(round);
j++;
}
}
return j;
}
std::string Silkscreen::CheckMySlot(int tid, uint64_t round, size_t k) const {
if (owner(k) != tid) return "";
const int v = *data(k);
const int w = static_cast<char>(round);
if (v == w) return "=";
return
Json("position", k) + ", " + Json("is", v) + ", " + Json("expected", w);
}
class Worker {
public:
// Does not take ownership of 'silkscreen'.
Worker(int pid, const std::vector<std::string> *words,
std::vector<int> tid_list, int tid, Silkscreen *silkscreen)
: pid_(pid), tid_(tid), words_(words), tid_list_(tid_list),
silkscreen_(silkscreen), rndeng_(std::random_device()()) {
}
~Worker() {}
void Run();
private:
static constexpr size_t kBufMin = 12;
#ifdef HAVE_FEATURE_MEMORY_SANITIZER
// Use smaller buffers if cpu_check is built with msan. Otherwise
// we will time out in testing.
static constexpr size_t kBufMax = 1 << 16; // 64 KiB
#else
static constexpr size_t kBufMax = 1 << 20; // 1 MiB
#endif
struct FloatingPointResults {
bool operator!=(const FloatingPointResults& other) const {
return d != other.d;
}
double d = 0.0;
};
typedef struct {
const char *name;
FloatingPointResults (Worker::*func)(
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer*) const;
} generatorItem;
struct BufferSet {
void Alloc(std::unique_ptr<MalignBuffer> *p) {
const size_t kBufCap = kBufMax + 23 * pagesize;
if (!*p) {
p->reset(new MalignBuffer(kBufCap));
}
}
std::unique_ptr<MalignBuffer> original;
std::unique_ptr<MalignBuffer> compressed;
std::unique_ptr<MalignBuffer> encrypted;
std::unique_ptr<MalignBuffer> copied;
std::unique_ptr<MalignBuffer> decrypted;
std::unique_ptr<MalignBuffer> decompressed;
std::unique_ptr<MalignBuffer> re_made;
};
FloatingPointResults FillBufferSystematic(
uint32_t unused_seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer* b) const;
FloatingPointResults FillBufferRandomData(
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer* b) const;
FloatingPointResults FillBufferRandomText(
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer* b) const;
FloatingPointResults FillBufferGrilledCheese(
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer* b) const;
void MadviseDontNeed(const MalignBuffer &s) const;
size_t Alignment();
void MaybeFlush(const MalignBuffer &s);
// Attempts to schedules CPU frequency using the Worker's.
// FVTController object. Returns the scheduled frequency or
// 0 if there is no FVTController available.
int ScheduledMHz() const;
MalignBuffer::CopyMethod CopyMethod();
std::string FVT() const;
MalignBuffer::PunchedHole PunchedHole(size_t bufsize);
FloatingPointResults GenerateData(
const generatorItem& generator,
const MalignBuffer::PunchedHole& hole,
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer* b) const;
// Emits a failure record.
// TODO: bump error count here, and maybe log, instead of at every
// call site.
std::string Jfail(const std::string &err, const std::string &v) {
if (errorCount > error_limit) {
exiting = true;
LOG(INFO) << "I am quitting after " << errorCount << " errors";
}
return "{ " + JsonRecord("fail", Json("err", err) + ", " + v) + ", " +
JTag() + " }";
}
// Array of random data generators.
static const std::vector<generatorItem> kGenerators;
const uint64_t pid_;
const int tid_;
const std::vector<std::string> *words_;
const std::vector<int> tid_list_;
Silkscreen* const silkscreen_;
// We don't really need "good" random numbers.
// std::mt19937_64 rndeng_;
std::knuth_b rndeng_;
uint64_t round_ = 0;
std::unique_ptr<FVTController> fvt_controller_;
};
const std::vector<Worker::generatorItem> Worker::kGenerators = {
{"SYSTEMATIC", &Worker::FillBufferSystematic},
{"DATA", &Worker::FillBufferRandomData},
{"TEXT", &Worker::FillBufferRandomText},
{"CHEESE", &Worker::FillBufferGrilledCheese},
};
std::string Worker::FVT() const {
if (fvt_controller_ == nullptr) return "";
return fvt_controller_->FVT();
}
int Worker::ScheduledMHz() const {
if (fvt_controller_ == nullptr) {
return 0;
}
if (fixed_min_frequency && (fixed_min_frequency == fixed_max_frequency)) {
// User-specified fixed frequency.
return fixed_min_frequency;
}
if (!do_freq_sweep && !do_freq_hi_lo
&& !fixed_min_frequency && !fixed_max_frequency) {
// Run at maximum frequency.
return fvt_controller_->limit_mHz();
}
const int low_f =
fixed_min_frequency ? fixed_min_frequency : fvt_controller_->kMinTurboMHz;
// hi_f cannot exceed limit
const int limit_mHz = fvt_controller_->limit_mHz();
const int hi_f = fixed_max_frequency
? std::min<int>(fixed_max_frequency, limit_mHz)
: limit_mHz;
int64_t t = TimeInSeconds() / seconds_per_freq;
if (do_freq_hi_lo) {
const int step = t % 2;
return step ? low_f : hi_f;
} else {
const int steps = 1 + (hi_f - low_f) / 100;
const int full_ramps = t / steps;
const bool upwards = full_ramps % 2;
const int step = t % steps;
return upwards ? low_f + 100 * step : hi_f - 100 * step;
}
}
// Fills 'b' with a systematic pattern.
// Returns iterate of chaotic floating point function of 'seed', with some
// reciprocal torture.
Worker::FloatingPointResults Worker::FillBufferSystematic(
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer* b) const {
// Format: 2 bytes of PID, 3 bytes of round number, 3 bytes of offset.
// Note: Perhaps should be AC-modulated. Perhaps should be absolute aligned
// for easier recognition.
// Note: appropriate for LE machines only.
FloatingPointResults fp;
fp.d = std::max<uint32_t>(seed, 2);
for (size_t i = 0; i * 8 < b->size(); i++) {
const size_t p = 8 * i;
const size_t k = std::min<size_t>(8, b->size() - p);
const uint64_t v =
((pid_ & 0xffff) << 48)
| ((round_ & 0xffffff) << 24) | (i & 0xffffff);
for (size_t m = 0; m < k; m++) {
(b->data())[p + m] = *(reinterpret_cast<const char*>(&v) + m);
}
fp.d = 1.0 / ChaoticF1(1.0 / fp.d);
}
fp.d = 1.0 / fp.d;
return fp;
}
Worker::FloatingPointResults Worker::FillBufferRandomData(
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer* b) const {
std::knuth_b rng(seed);
std::uniform_int_distribution<uint64_t>
dist(0, std::numeric_limits<uint64_t>::max());
size_t p = 0;
const size_t length = b->size();
// Repeatedly append random number (one to eight) random bytes.
while (p < length) {
const size_t max_span = std::min<size_t>(length - p, 8);
const size_t z = std::uniform_int_distribution<size_t>(1, max_span)(rng);
const uint64_t v = dist(rng);
b->CopyFrom(p, reinterpret_cast<const char*>(&v), z, copy_method);
p += z;
}
return FloatingPointResults();
}
Worker::FloatingPointResults Worker::FillBufferRandomText(
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos, MalignBuffer* b) const {
std::knuth_b rng(seed);
std::exponential_distribution<double> dist(20);
const size_t bufsize = b->size();
size_t pos = 0;
while (pos < bufsize) {
const size_t r = std::min(static_cast<size_t>(dist(rng) * words_->size()),
words_->size() - 1);
const auto &word = (*words_)[r];
const size_t wordlen = word.size();
if (pos + wordlen >= bufsize) {
break;
}
b->CopyFrom(pos, word.c_str(), wordlen, copy_method);
pos += wordlen;
if (pos < bufsize) {
b->Memset(pos, ' ', 1, use_repstos);
pos++;
}
}
// Pad with spaces
b->Memset(pos, ' ', bufsize - pos, use_repstos);
return FloatingPointResults();
}
// memset (conventional or rep;stos) randomly aligned, random width, randomly
// overlapped stretches of buffer. Constants aim to hit multiple times in cache
// lines and buffers. Untuned and based on nothing but hunches.
Worker::FloatingPointResults Worker::FillBufferGrilledCheese(
uint32_t seed, MalignBuffer::CopyMethod copy_method, bool use_repstos,
MalignBuffer* b) const {
std::knuth_b rng(seed);
const size_t kAdvance = 15;
const size_t kWindow = 64;
unsigned char flavor = 0;
b->Memset(0, 0, b->size(), use_repstos);
for (int base = kWindow; base < b->size(); base += kAdvance) {
if (std::uniform_int_distribution<int>(0, 1)(rng)) continue;
flavor++;
const size_t start =
std::uniform_int_distribution<size_t>(base - kWindow, base)(rng);
const size_t end = std::uniform_int_distribution<int>(start, base)(rng);
b->Memset(start, flavor, 1 + end - start, use_repstos);
}
return FloatingPointResults();
}
Worker::FloatingPointResults Worker::GenerateData(
const generatorItem& generator,
const MalignBuffer::PunchedHole& hole,
uint32_t seed, MalignBuffer::CopyMethod copy_method,
bool use_repstos,
MalignBuffer* b) const {
const FloatingPointResults f =
(this->*generator.func)(seed, copy_method, use_repstos, b);
b->PunchHole(hole, use_repstos);
return f;
}
// Hints to the OS to release the buffer's memory.
void Worker::MadviseDontNeed(const MalignBuffer &s) const {
// Round up the buffer start address to a page boundary.
intptr_t start = ((intptr_t) s.data() + pagesize - 1) & ~(pagesize - 1);
// Round down the buffer end address to a page boundary.
intptr_t end = ((intptr_t) (s.data() + s.size() - 1)) & ~(pagesize - 1);
if (end - start >= pagesize) {
if (madvise((char *)start, end - start, MADV_DONTNEED) == -1) {
LOG(WARN) << "tid " << tid_
<< " madvise(MADV_DONTNEED) failed: " << strerror(errno);
}
}
}
void Worker::MaybeFlush(const MalignBuffer& s) {
// Half the time, tell the OS to release the destination buffer.
if (do_flush && std::uniform_int_distribution<int>(0, 1)(rndeng_)) {
s.RandomFlush(&rndeng_);
}
}
size_t Worker::Alignment() {
return do_misalign ?
std::uniform_int_distribution<size_t>(0, pagesize)(rndeng_) : 0;
}
MalignBuffer::CopyMethod Worker::CopyMethod() {
std::vector<MalignBuffer::CopyMethod> v;
v.push_back(MalignBuffer::kMemcpy);
if (do_repmovsb) {
// Weight rep;mov more heavily.
for (int i = 0; i < 3; i++) {
v.push_back(MalignBuffer::kRepMov);
}
}
if (do_sse_128_memcpy) v.push_back(MalignBuffer::kSseBy128);
if (do_avx_256_memcpy) v.push_back(MalignBuffer::kAvxBy256);
if (do_avx_512_memcpy) v.push_back(MalignBuffer::kAvxBy512);
size_t k = std::uniform_int_distribution<int>(0, v.size() - 1)(rndeng_);
return v[k];
}
MalignBuffer::PunchedHole Worker::PunchedHole(size_t bufsize) {
MalignBuffer::PunchedHole hole;
hole.length =
std::uniform_int_distribution<size_t>(
1, std::min<size_t>(bufsize, 8192))(rndeng_);
hole.start =
std::uniform_int_distribution<size_t>(
0, bufsize - hole.length)(rndeng_);
return hole;
}
std::string OpenSSL_Hash(const MalignBuffer &s, const EVP_MD *type) {
EVP_MD_CTX *ctx;
ctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(ctx, type, nullptr);
std::string hash;
hash.resize(EVP_MD_CTX_size(ctx));
InitializeMemoryForSanitizer(hash.data(), EVP_MD_CTX_size(ctx));
EVP_DigestUpdate(ctx, s.data(), s.size());
EVP_DigestFinal_ex(ctx, (uint8_t *)&hash[0], nullptr);
EVP_MD_CTX_destroy(ctx);
return HexStr(hash);
}
void Worker::Run() {
// Array of hash/checksum routines.
typedef struct {
const char *name;
std::string (*func)(const MalignBuffer &);
} hashItem;
std::vector<hashItem> hashers = {
{
"MD5",
[](const MalignBuffer &s) -> std::string {
return OpenSSL_Hash(s, EVP_md5());
},
},
{
"SHA1",
[](const MalignBuffer &s) -> std::string {
return OpenSSL_Hash(s, EVP_sha1());
},
},
{
"SHA256",
[](const MalignBuffer &s) -> std::string {
return OpenSSL_Hash(s, EVP_sha256());
},
},
{
"SHA512",
[](const MalignBuffer &s) -> std::string {
return OpenSSL_Hash(s, EVP_sha512());
},
},
{
"ADLER32", // exported by zlib
[](const MalignBuffer &s) -> std::string {
uLong c = adler32(0, Z_NULL, 0);
c = adler32(c, (const Bytef *)s.data(), s.size());
return HexData((const char *)&c, sizeof(c));
},
},
{
"CRC32", // exported by zlib.
[](const MalignBuffer &s) -> std::string {
uLong c = crc32(0, Z_NULL, 0);
c = crc32(c, (const Bytef *)s.data(), s.size());
return HexData((const char *)&c, sizeof(c));
},
},
{
"CRC32C", // crc32 instruction on SSSE3
[](const MalignBuffer &s) -> std::string {
uint32_t c = crc32c(s.data(), s.size());
return HexData((const char *)&c, sizeof(c));
},
},
{
"FarmHash64", // Google farmhash
[](const MalignBuffer &s) -> std::string {
uint64_t c = util::Hash64(s.data(), s.size());
return HexData((const char *)&c, sizeof(c));
},
},
};
// Array of compression routines.
typedef struct {
const char *name;
int (*enc)(MalignBuffer *, const MalignBuffer &);
int (*dec)(MalignBuffer *, const MalignBuffer &);
} compressorItem;
std::vector<compressorItem> compressors = {
{
"ZLIB",
[](MalignBuffer *o, const MalignBuffer &s) {
uLongf olen = compressBound(s.size());
o->resize(olen);
int err = compress2((Bytef *)o->data(), &olen, (Bytef *)s.data(),
s.size(), Z_BEST_SPEED);
if (err != Z_OK) {
LOG(DEBUG) << "zlib compression failed: " << err
<< " srclen: " << s.size()
<< " destlen: " << o->size();
return err;
}
o->resize(olen);
return 0;
},
[](MalignBuffer *o, const MalignBuffer &s) {
uLongf olen = o->size();
int err = uncompress((Bytef *)o->data(), &olen, (Bytef *)s.data(),
s.size());
if (err != Z_OK) {
LOG(DEBUG) << "zlib decompression failed: " << err
<< " srclen: " << s.size()
<< " destlen: " << o->size();
return err;
}
o->resize(olen);
return 0;
},
},
};
// Choose generator and compressor uniformly.
std::uniform_int_distribution<int> gen_dist(
0, do_provenance ? 0 : kGenerators.size() - 1);
auto Gen = std::bind(gen_dist, rndeng_);
std::uniform_int_distribution<int> comp_dist(0, compressors.size() - 1);
auto Comp = std::bind(comp_dist, rndeng_);
// Run one randomly-chosen hash routine each round.
size_t hash_choice;
std::string hash_value;
EVP_CIPHER_CTX *cipher_ctx;
cipher_ctx = EVP_CIPHER_CTX_new();
// Creates FVT controller if we can do so.
if (do_fvt) {
fvt_controller_ = FVTController::Create(tid_);
fvt_controller_->SetCurrentFreqLimitMhz(fvt_controller_->limit_mHz());
fvt_controller_->ControlFastStringOps(do_fast_string_ops);
LOG(INFO) << "Tid: " << tid_
<< " Enables: " << fvt_controller_->InterestingEnables();
}
// MalignBuffers are allocated once if !do_madvise. Otherwise they are
// reallocated each iteration of the main loop, creating much more memory
// allocator work, which may itself exhibit, and suffer from, CPU defects.
std::unique_ptr<BufferSet> b;
uint64_t expected_slot_count = 0;
Avx avx;
while (!exiting) {
if (std::thread::hardware_concurrency() > 1) {
if (!SetAffinity(tid_)) {
LOG(WARN) << "Couldnt run on " << tid_ << " sleeping a bit";
sleep(30);
continue;
}
}
round_++;
if (!b) {
b.reset(new BufferSet);
}
if (do_noise) {
NoiseScheduler::BlockUntilOn();
}
const int turbo_mhz = ScheduledMHz(); // 0 if no FVT.
if (fvt_controller_ != nullptr) {
fvt_controller_->SetCurrentFreqLimitMhz(turbo_mhz);
fvt_controller_->MonitorFrequency();
}
auto Turbo = [&turbo_mhz](){
return Json("turbo", turbo_mhz);
};
auto Tid = [this](int tid) {
return "{ " + Json("tid", tid) + (do_fvt ? ", " + FVT() : "") + " }";
};
auto Writer = [this, Tid](){
return "\"writer\": " + Tid(tid_);
};
LOG_EVERY_N_SECS(INFO, 30) << Jstat(
Json("elapsed_s", static_cast<uint64_t>(TimeInSeconds() - t0)) + ", " +
Json("failures", errorCount.load()) + ", " +
Json("successes", successCount.load()) + ", " + Writer() +
(fvt_controller_ != nullptr
? ", " + Json("meanFreq", fvt_controller_->GetMeanFreqMhz()) +
", " + Json("maxFreq", fvt_controller_->max_mHz())
: ""));
if (do_avx_heavy) {
const std::string e = avx.MaybeGoHot();
if (!e.empty()) {
LOG(ERROR) << Jfail(e, Writer() + ", " + Turbo());
errorCount++;
}
}
#ifdef USE_BORINGSSL
if (do_ssl_self_check && BORINGSSL_self_test() == 0) {
LOG(ERROR) << Jfail("BORINGSSL_self_test", Writer() + ", " + Turbo());
errorCount++;
continue;
}
#endif
const bool madvise =
do_madvise && std::uniform_int_distribution<int>(0, 1)(rndeng_);
const size_t bufsize =
std::uniform_int_distribution<size_t>(kBufMin, kBufMax)(rndeng_);
auto &gen = kGenerators[Gen()];
auto &comp = compressors[Comp()];
const MalignBuffer::PunchedHole hole = PunchedHole(bufsize);
const MalignBuffer::CopyMethod copy_method = CopyMethod();
const bool use_repstos =
do_repstosb && std::uniform_int_distribution<int>(0, 1)(rndeng_);
auto BlockSummary = [&]() {
std::stringstream block_summary;
block_summary
<< Json("pattern", gen.name) << ", "
<< Json("copy", MalignBuffer::ToString(copy_method)) << ", "
<< Json("memset", use_repstos ? "rep;sto" : "memset") << ", "
<< JsonBool("madvise", madvise) << ", "
<< Json("size", bufsize) << ", "
<< Json("pid", pid_) << ", "
<< Json("round", round_) << ", "
<< hole.ToString();
return block_summary.str();
};
auto WriterInfo = [&](){
return Writer() + ", " + Turbo() + ", " + BlockSummary();
};
if (round_ > 1) {
uint64_t slots_read = 0;
uint64_t errs_this_round = 0;
const uint64_t kErrLimit = 20;
for (size_t k = 0; k < Silkscreen::kSize; k++) {
const std::string err = silkscreen_->CheckMySlot(tid_, round_ - 1, k);
if (!err.empty()) {
slots_read++;
if (err != "=") {
errs_this_round++;
if (errs_this_round <= kErrLimit) {
errorCount++;
LOG(ERROR) << Jfail("Silkscreen", JsonRecord("syndrome", err) +
", " + WriterInfo());
} else {
// When Silkscreen fails, it often fails, on several bad machines,
// in a surprising way: all slots owned by reading tid are
// are corrupt, in a way that suggests the previous round's
// writes never happened. Weird, and deserves some study, but
// meanwhile the log spew is suppressed.
}
}
}
}
if (errs_this_round > kErrLimit) {
LOG(ERROR) << Jfail(
"Silkscreen",
JsonRecord("syndrome", Json("many_errors", errs_this_round) + ", " +
Json("slots_read", slots_read)) +
", " + WriterInfo());
errorCount++;
}
if (expected_slot_count != slots_read) {
LOG(ERROR) << Jfail("Silkscreen",
Json("read", slots_read) + ", " +
Json("expected", expected_slot_count) + ", " +
WriterInfo());
errorCount++;
}
}
const uint64_t slots_written = silkscreen_->WriteMySlots(tid_, round_);
if (!expected_slot_count) {
expected_slot_count = slots_written;
}
if (expected_slot_count != slots_written) {
LOG(ERROR) << Jfail("Silkscreen",
Json("written", slots_written) + ", " +
Json("expected", expected_slot_count) + ", " +
WriterInfo());
errorCount++;
}
if (do_avx_heavy) {
// If we tried to do AVX heavy stuff. Try to run AVX heavy again to try
// to spike current.
const std::string e = avx.BurnIfAvxHeavy();
if (!e.empty()) {
LOG(ERROR) << Jfail(e, Writer() + ", " + Turbo());
errorCount++;
}
}
const auto buffer_seed = rndeng_();
if (!b->original) b->Alloc(&b->original);
b->original->Initialize(Alignment(), bufsize);
const FloatingPointResults f =
GenerateData(gen, hole, buffer_seed,
copy_method, use_repstos, b->original.get());
MaybeFlush(*b->original);
MalignBuffer* head = b->original.get();
if (do_hashes) {
hash_choice =
std::uniform_int_distribution<size_t>(0, hashers.size() - 1)(rndeng_);
hash_value = hashers[hash_choice].func(*head);
}
if (do_compress) {
// Run our randomly chosen compressor.
if (!b->compressed) b->Alloc(&b->compressed);
b->compressed->Initialize(Alignment(), bufsize);
MaybeFlush(*b->compressed);
const int err = comp.enc(b->compressed.get(), *head);
LOG(DEBUG) << WriterInfo()
<< " original->size(): " << head->size()
<< ", compressed.size(): " << b->compressed->size() << ".";
if (err) {
LOG(ERROR) << Jfail("Compression",
Json("syndrome", err) + ", " + WriterInfo());
errorCount++;
continue;
}
MaybeFlush(*b->compressed);
head = b->compressed.get();
LOG(DEBUG) << WriterInfo() << "compress done.";
if (exiting) break;
}
const unsigned char key[33] = "0123456789abcdef0123456789abcdef";
const std::string ivec(b->original->data(), kBufMin);
unsigned char gmac[16];
const MalignBuffer* const unencrypted = head;
if (do_encrypt) {
// Encrypt.
if (!b->encrypted) b->Alloc(&b->encrypted);
b->encrypted->Initialize(Alignment(), head->size());
MaybeFlush(*b->encrypted);
int enc_len = 0, enc_unused_len = 0;
EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key,
(unsigned char *)ivec.data(), 1);
if (madvise) MadviseDontNeed(*b->encrypted);
if (EVP_CipherUpdate(
cipher_ctx, (unsigned char *)b->encrypted->data(), &enc_len,
(unsigned char *)head->data(), head->size()) != 1) {
LOG(ERROR) << Jfail("EVP_CipherUpdate", WriterInfo());
errorCount++;
EVP_CIPHER_CTX_cleanup(cipher_ctx);
continue;
}
if (EVP_CipherFinal_ex(cipher_ctx, nullptr, &enc_unused_len) != 1) {
LOG(ERROR) << Jfail("encrypt_EVP_CipherFinal_ex", WriterInfo());
errorCount++;
EVP_CIPHER_CTX_cleanup(cipher_ctx);
continue;
}
enc_len += enc_unused_len;
if (enc_len != (int)b->encrypted->size()) {
std::stringstream ss;
ss << "enc_length: " << enc_len << " vs: " << b->encrypted->size();
LOG(ERROR) << Jfail("encrypt_length_mismatch",
Json("syndrome", ss.str()) + ", " + WriterInfo());
errorCount++;
EVP_CIPHER_CTX_cleanup(cipher_ctx);
continue;
}
if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_GET_TAG, sizeof(gmac),
gmac) != 1) {
LOG(ERROR) << Jfail("EVP_CTRL_GCM_GET_TAG", WriterInfo());
errorCount++;
EVP_CIPHER_CTX_cleanup(cipher_ctx);
continue;
}
EVP_CIPHER_CTX_cleanup(cipher_ctx);
MaybeFlush(*b->encrypted);
head = b->encrypted.get();
LOG(DEBUG) << "Encrypt done " << WriterInfo();
if (exiting) break;
}
// Make a copy.
if (!b->copied) b->Alloc(&b->copied);
b->copied->Initialize(Alignment(), head->size());
MaybeFlush(*b->copied);
if (madvise) MadviseDontNeed(*b->copied);
std::string syndrome = b->copied->CopyFrom(*head, copy_method);
if (!syndrome.empty()) {
LOG(ERROR) << Jfail(
"writer-detected-copy",
JsonRecord("syndrome", syndrome) + ", " + WriterInfo());
errorCount++;
continue;
}
MaybeFlush(*b->copied);
// Switch to an alternate CPU.
int newcpu = tid_;
if (do_hop && std::thread::hardware_concurrency() > 1) {
std::vector<int> cpus;
for (int i : tid_list_) {
if (i == tid_) continue;
cpus.push_back(i);
}
if (!cpus.empty()) {
int cpuoff =
std::uniform_int_distribution<int>(0, cpus.size() - 1)(rndeng_);
newcpu = cpus[cpuoff];
cpus.erase(cpus.begin() + cpuoff);
if (!SetAffinity(newcpu)) {
// Tough luck, can't run on chosen CPU.
// Validate on same cpu we were on.
newcpu = tid_;
}
}
}
auto Reader = [&Tid, &newcpu](){
return "\"reader\": " + Tid(newcpu);
};
auto WriterReaderInfo = [&](){
return
Writer() + ", " + Reader() + ", " + Turbo() + ", " + BlockSummary();
};
// Re-verify buffer copy
syndrome = b->copied->Syndrome(*head);
if (!syndrome.empty()) {
LOG(ERROR) << Jfail(
"copy", JsonRecord("syndrome", syndrome) + ", " + WriterInfo());
errorCount++;
continue;
}
MaybeFlush(*b->copied);
head = b->copied.get();
if (do_encrypt) {
// Decrypt.
if (!b->decrypted) b->Alloc(&b->decrypted);
b->decrypted->Initialize(Alignment(), head->size());
MaybeFlush(*b->decrypted);
int dec_len = 0, dec_extra_len = 0;
EVP_CipherInit_ex(cipher_ctx, EVP_aes_256_gcm(), NULL, key,
(unsigned char *)ivec.data(), 0);
if (madvise) MadviseDontNeed(*b->decrypted);
if (EVP_CIPHER_CTX_ctrl(cipher_ctx, EVP_CTRL_GCM_SET_TAG, sizeof(gmac),
gmac) != 1) {
LOG(ERROR) << Jfail("EVP_CTRL_GCM_SET_TAG", WriterReaderInfo());
errorCount++;
EVP_CIPHER_CTX_cleanup(cipher_ctx);
continue;
}
if (EVP_CipherUpdate(
cipher_ctx, (unsigned char *)b->decrypted->data(), &dec_len,
(unsigned char *)head->data(), head->size()) != 1) {
LOG(ERROR) << Jfail("decryption", WriterReaderInfo());
errorCount++;
EVP_CIPHER_CTX_cleanup(cipher_ctx);
continue;
}
if (EVP_CipherFinal_ex(
cipher_ctx, (unsigned char *)(b->decrypted->data() + dec_len),
&dec_extra_len) != 1) {
LOG(ERROR) << Jfail("decrypt_EVP_CipherFinal_ex", WriterReaderInfo());
errorCount++;
EVP_CIPHER_CTX_cleanup(cipher_ctx);
continue;
}
dec_len += dec_extra_len;
EVP_CIPHER_CTX_cleanup(cipher_ctx);
if (dec_len != (int)b->decrypted->size()) {
std::stringstream ss;
ss << "dec_length: " << dec_len << " vs: " << b->decrypted->size();
LOG(ERROR) << Jfail("decrypt_length_mismatch",
Json("syndrome", ss.str()) + ", " +
WriterReaderInfo());
errorCount++;
continue;
}
MaybeFlush(*b->decrypted);
head = b->decrypted.get();
syndrome = unencrypted->Syndrome(*head);
if (!syndrome.empty()) {
LOG(ERROR) << Jfail(
"decryption_mismatch",
JsonRecord("syndrome", syndrome) + ", " + WriterReaderInfo());
errorCount++;
continue;
}
LOG(DEBUG) << WriterReaderInfo() << " decrypt done";
if (exiting) break;
}
if (do_compress) {
// Run decompressor.
if (!b->decompressed) b->Alloc(&b->decompressed);
b->decompressed->Initialize(Alignment(), bufsize);
MaybeFlush(*b->decompressed);
if (madvise) MadviseDontNeed(*b->decompressed);
const int err = comp.dec(b->decompressed.get(), *head);
if (err) {
LOG(ERROR) << Jfail("uncompression",
Json("syndrome", err) + ", " + WriterReaderInfo());
errorCount++;
continue;
}
if (b->decompressed->size() != bufsize) {
std::stringstream ss;
ss << "dec_length: " << b->decompressed->size() << " vs: " << bufsize;
LOG(ERROR) << Jfail(
"decompressed_size",
Json("syndrome", ss.str()) + ", " + WriterReaderInfo());
errorCount++;
continue;
}
MaybeFlush(*b->decompressed);
head = b->decompressed.get();
LOG(DEBUG) << WriterReaderInfo() << " uncompress done";
}
if (!b->re_made) b->Alloc(&b->re_made);
b->re_made->Initialize(Alignment(), bufsize);
const FloatingPointResults f_r =
GenerateData(gen, hole, buffer_seed,
copy_method, use_repstos, b->re_made.get());
syndrome = b->original->Syndrome(*b->re_made);
if (!syndrome.empty()) {
LOG(ERROR) << Jfail("re-make", JsonRecord("syndrome", syndrome) + ", " +
WriterReaderInfo());
errorCount++;
continue;
}
if (f != f_r) {
std::stringstream ss;
ss << "Was: " << f.d << " Is: " << f_r.d;
LOG(ERROR) << Jfail(
"fp-double", Json("syndrome", ss.str()) + ", " + WriterReaderInfo());
errorCount++;
continue;
}
if (do_hashes) {
// Re-run hash func.
std::string hash = hashers[hash_choice].func(*head);
if (hash_value != hash) {
std::stringstream ss;
ss << "hash was: " << hash_value << " is: " << hash;
LOG(ERROR) << Jfail(
"hash", Json("syndrome", ss.str()) + ", " + WriterReaderInfo());
errorCount++;
continue;
}
LOG(DEBUG) << WriterReaderInfo() << " rehash done";
}
// Release MalignBuffer memory allocations.
if (do_madvise) b.reset(nullptr);
successCount++;
}
EVP_CIPHER_CTX_free(cipher_ctx);
LOG(INFO) << "tid " << tid_ << " exiting.";
}
static void UsageIf(bool v) {
if (!v) return;
LOG(ERROR) << "Usage cpu_check [-a] [-b] [-c] [-d] [-e] [-F] [-h]"
<< " [-m] [-nN] [-p] [-qNNN] [-r] [-x] [-X] [-s] [-Y] [-H] [-kXXX]"
<< " [-z] [-cn,n,...,n] [-fMinF[-MaxF]] [-tNNN] [-u]"
<< "\n a: Do not misalign"
<< "\n b: Do not run BoringSSL self check"
<< "\n c: Explicit list of CPUs"
<< "\n d: Do not rep stosb"
<< "\n e: Do not encrypt"
<< "\n f: Fixed specified turbo frequency (multiple of 100)"
<< "\n g: Do not touch frequency, voltage and thermal controls"
<< "\n F: Randomly flush caches (inverted option)"
<< "\n h: Do not hash"
<< "\n m: Do not madvise, do not malloc per iteration"
<< "\n l: Do not provoke heavy AVX power fluctuations"
<< "\n n: Generate noise"
<< "\n N: Generate noise, invert -c"
<< "\n p: Corrupt data provenance"
<< "\n q: Quit if more than N errors"
<< "\n r: Do not repmovsb"
<< "\n t: Timeout in seconds"
<< "\n x: Do not use AVX:256"
<< "\n X: Do use AVX512"
<< "\n s: Do not switch CPUs for verification"
<< "\n u: Do not use fast string ops"
<< "\n Y: Do frequency sweep"
<< "\n H: Slam between low and high frequency"
<< "\n k: Frequency step period (default 300)"
<< "\n z: Do not compress/uncompress";
exit(2);
}
int main(int argc, char **argv) {
std::vector<int> tid_list;
int64_t timeout = 0;
#ifdef IN_GOOGLE3
// Initialize the symbolizer to get a human-readable stack trace.
absl::InitializeSymbolizer(argv[0]);
absl::FailureSignalHandlerOptions options;
absl::InstallFailureSignalHandler(options);
#endif
for (int i = 1; i < argc; i++) {
const char *flag = argv[i];
UsageIf(flag[0] != '-');
for (flag++; *flag != 0; flag++) {
switch (*flag) {
case 'a':
do_misalign = false;
break;
case 'b':
do_ssl_self_check = false;
break;
case 'c':
{
std::string c = "";
for (flag++; *flag != 0; flag++) {
c += *flag;
}
std::stringstream s(c);
int t;
while (s >> t) {
tid_list.push_back(t);
if (s.peek() == ',') s.ignore();
}
}
break;
case 'd':
do_repstosb = false;
break;
case 'e':
do_encrypt = false;
break;
case 'f':
{
std::string c(++flag);
flag += c.length();
std::stringstream s(c);
s >> fixed_min_frequency;
fixed_max_frequency = fixed_min_frequency;
if (s.get() == '-') {
s >> fixed_max_frequency;
do_freq_sweep = true;
}
}
break;
case 'F':
do_flush = true;
break;
case 'g':
do_fvt = false;
break;
case 'H':
do_freq_hi_lo = true;
break;
case 'h':
do_hashes = false;
break;
case 'l':
do_avx_heavy = false;
break;
case 'm':
do_madvise = false;
break;
case 'n':
do_noise = true;
break;
case 'N':
do_noise = true;
do_invert_cores = true;
do_encrypt = false;
do_hashes = false;
do_compress = false;
do_madvise = false;
do_hop = false;
break;
case 'p':
do_provenance = true;
do_encrypt = false;
do_hashes = false;
do_compress = false;
break;
case 'q':
{
std::string c(++flag);
flag += c.length();
std::stringstream s(c);
s >> error_limit;
}
break;
case 'r':
do_repmovsb = false;
break;
case 's':
do_hop = false;
break;
case 'u':
do_fast_string_ops = false;
break;
case 'x':
do_avx_256_memcpy = false;
break;
case 'X':
do_avx_512_memcpy = true;
break;
case 'Y':
do_freq_sweep = true;
break;
case 'k':
{
std::string c(++flag);
flag += c.length();
std::stringstream s(c);
s >> seconds_per_freq;
}
break;
case 't':
{
std::string c(++flag);
flag += c.length();
std::stringstream s(c);
s >> timeout;
}
break;
case 'z':
do_compress = false;
break;
default:
UsageIf(true);
}
if (*flag == 0) break;
}
}
LOG(INFO) << "Starting " << argv[0] << " version " cpu_check_VERSION
<< (do_misalign ? "" : " No misalign ")
<< (do_repstosb ? "" : " No repstosb")
<< (!do_flush ? "" : " Cache-line flush ")
<< (do_encrypt ? "" : " No encryption ")
<< (do_hashes ? "" : " No hash ")
<< (do_madvise ? "" : " No madvise ")
<< (do_provenance ? " Provenance " : "")
<< (do_repmovsb ? "" : " No repmovsb ")
<< (do_sse_128_memcpy ? "" : " No SSE:128 ")
<< (do_avx_256_memcpy ? "" : " No AVX:256 ")
<< (do_avx_512_memcpy ? "" : " No AVX:512 ")
<< (do_avx_heavy ? " AVX_heavy " : "")
<< (do_compress ? "" : " No compression ")
<< (do_hop ? "" : " No thread_switch ")
<< (do_ssl_self_check ? "" : " No BoringSSL self check")
<< (!do_freq_sweep ? "" : " FrequencySweep ")
<< (!do_freq_hi_lo ? "" : " FreqHiLo ")
<< (do_noise ? " NOISE" : "")
<< (do_fast_string_ops ? "" : " No FastStringOps");
std::vector<std::thread *> threads;
std::vector<Worker *> workers;
std::vector<std::string> words = ReadDict();
if (words.empty()) {
LOG(ERROR) << "No word list found.";
exit(1);
}
int cpus = std::thread::hardware_concurrency();
LOG(INFO) << "Detected hardware concurrency: " << cpus;
if (do_invert_cores) {
// Clumsily complement the tid_list in -N mode.
std::vector<int> v;
for (int i = 0; i < cpus; i++) {
if (std::find(tid_list.begin(), tid_list.end(), i) == tid_list.end()) {
v.push_back(i);
}
}
tid_list = v;
}
if (tid_list.empty()) {
for (int i = 0; i < cpus; i++) {
tid_list.push_back(i);
}
} else {
for (int t : tid_list) {
LOG(INFO) << "Explicitly testing cpu: " << t;
}
}
// Silkscreen instance shared by all threads.
Silkscreen silkscreen(tid_list);
for (int tid : tid_list) {
workers.push_back(new Worker(getpid(), &words, tid_list, tid, &silkscreen));
threads.push_back(new std::thread(&Worker::Run, workers.back()));
}
signal(SIGTERM, [](int) { exiting = true; });
signal(SIGINT, [](int) { exiting = true; });
struct timeval last_cpu = {0, 0};
double last_time = t0;
while (!exiting) {
sleep(60);
struct rusage ru;
double secs = TimeInSeconds();
if (timeout > 0 && secs >= t0 + timeout) {
exiting = true;
}
double secondsPerError = (secs - t0) / errorCount.load();
if (getrusage(RUSAGE_SELF, &ru) == -1) {
LOG(ERROR) << "getrusage failed: " << strerror(errno);
} else {
float cpu = (((ru.ru_utime.tv_sec - last_cpu.tv_sec) * 1000000.0) +
(ru.ru_utime.tv_usec - last_cpu.tv_usec)) /
1000000.0;
LOG(INFO) << "Errors: " << errorCount.load()
<< " Successes: " << successCount.load()
<< " CPU " << cpu / (secs - last_time) << " s/s"
<< " Seconds Per Error: " << secondsPerError;
last_cpu = ru.ru_utime;
}
last_time = secs;
}
// shutting down.
for (auto &t : threads) {
t->join();
delete t;
}
for (auto w : workers) {
delete w;
}
LOG(ERROR) << errorCount.load() << " ERRORS, " << successCount.load()
<< " SUCCESSES.";
LOG(INFO) << "Exiting.";
exit(errorCount != 0);
}
View Git Blame Copy Permalink