refactor fatal call (#511)

* refactor fatal handling
This commit is contained in:
Kjell Hedström - seeking Senior Engineering roles as well as contract opportunities 2023-12-05 21:04:01 -07:00 committed by GitHub
parent d70ae50ee3
commit bba825815a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 48 additions and 44 deletions

BIN
.DS_Store vendored Normal file

Binary file not shown.

BIN
docs/.DS_Store vendored Normal file

Binary file not shown.

View File

@ -148,45 +148,48 @@ namespace g3 {
message.get()->setExpression(boolean_expression); message.get()->setExpression(boolean_expression);
if (internal::wasFatal(level)) { if (internal::wasFatal(level)) {
auto fatalhook = g_fatal_pre_logging_hook; saveFatalMessage(level, stack_trace, message, fatal_signal);
// In case the fatal_pre logging actually will cause a crash in its turn
// let's not do recursive crashing!
setFatalPreLoggingHook(g_pre_fatal_hook_that_does_nothing);
++g_fatal_hook_recursive_counter; // thread safe counter
// "benign" race here. If two threads crashes, with recursive crashes
// then it's possible that the "other" fatal stack trace will be shown
// that's OK since it was anyhow the first crash detected
static const std::string first_stack_trace = stack_trace;
fatalhook();
message.get()->write().append(stack_trace);
if (g_fatal_hook_recursive_counter.load() > 1) {
message.get()->write().append(
"\n\n\nWARNING\n"
"A recursive crash detected. It is likely the hook set with 'setFatalPreLoggingHook(...)' is responsible\n\n")
.append("---First crash stacktrace: ")
.append(first_stack_trace)
.append("\n---End of first stacktrace\n");
}
FatalMessagePtr fatal_message{std::make_unique<FatalMessage>(*(message._move_only.get()), fatal_signal)};
// At destruction, flushes fatal message to g3LogWorker
// either we will stay here until the background worker has received the fatal
// message, flushed the crash message to the sinks and exits with the same fatal signal
//..... OR it's in unit-test mode then we throw a std::runtime_error (and never hit sleep)
fatalCall(fatal_message);
} else { } else {
pushMessageToLogger(message); pushMessageToLogger(message);
} }
} }
void saveFatalMessage(const LEVELS& level, const char* stack_trace, g3::LogMessagePtr& message, int& fatal_signal) {
auto fatalhook = g_fatal_pre_logging_hook;
// In case the fatal_pre logging actually will cause a crash in its turn
// let's not do recursive crashing!
setFatalPreLoggingHook(g_pre_fatal_hook_that_does_nothing);
++g_fatal_hook_recursive_counter; // thread safe counter
// "benign" race here. If two threads crashes, with recursive crashes
// then it's possible that the "other" fatal stack trace will be shown
// that's OK since it was anyhow the first crash detected
static const std::string first_stack_trace = stack_trace;
fatalhook();
message.get()->write().append(stack_trace);
if (g_fatal_hook_recursive_counter.load() > 1) {
message.get()->write().append(
"\n\n\nWARNING\n"
"A recursive crash detected. It is likely the hook set with 'setFatalPreLoggingHook(...)' is responsible\n\n")
.append("---First crash stacktrace: ")
.append(first_stack_trace)
.append("\n---End of first stacktrace\n");
}
FatalMessagePtr fatal_message{std::make_unique<FatalMessage>(*(message._move_only.get()), fatal_signal)};
// At destruction, flushes fatal message to g3LogWorker
// either we will stay here until the background worker has received the fatal
// message, flushed the crash message to the sinks and exits with the same fatal signal
//..... OR it's in unit-test mode then we throw a std::runtime_error (and never hit sleep)
fatalCall(fatal_message);
}
/** /**
* save the message to the logger. In case of called before the logger is instantiated * save the message to the logger. In case of called before the logger is instantiated
* the first message will be saved. Any following subsequent uninitialized log calls * the first message will be saved. Any following subsequent uninitialized log calls
* will be ignored. * will be ignored.
* *
* The first initialized log entry will also save the first uninitialized log message, if any * The first initialized log entry will also save the first uninitialized log message, if any
* @param log_entry to save to logger * @param log_entry to save to logger
*/ */
void pushMessageToLogger(LogMessagePtr incoming) { // todo rename to Push SavedMessage To Worker void pushMessageToLogger(LogMessagePtr incoming) { // todo rename to Push SavedMessage To Worker
// Uninitialized messages are ignored but does not CHECK/crash the logger // Uninitialized messages are ignored but does not CHECK/crash the logger
if (!internal::isLoggingInitialized()) { if (!internal::isLoggingInitialized()) {
@ -196,9 +199,8 @@ namespace g3 {
err.append(g_first_uninitialized_msg->message()); err.append(g_first_uninitialized_msg->message());
std::string& str = g_first_uninitialized_msg->write(); std::string& str = g_first_uninitialized_msg->write();
str.clear(); str.clear();
str.append(err); // replace content str.append(err); // replace content
std::cerr << str << std::endl; std::cerr << str << std::endl; });
});
return; return;
} }
@ -207,10 +209,10 @@ namespace g3 {
} }
/** Fatal call saved to logger. This will trigger SIGABRT or other fatal signal /** Fatal call saved to logger. This will trigger SIGABRT or other fatal signal
* to exit the program. After saving the fatal message the calling thread * to exit the program. After saving the fatal message the calling thread
* will sleep forever (i.e. until the background thread catches up, saves the fatal * will sleep forever (i.e. until the background thread catches up, saves the fatal
* message and kills the software with the fatal signal. * message and kills the software with the fatal signal.
*/ */
void pushFatalMessageToLogger(FatalMessagePtr message) { void pushFatalMessageToLogger(FatalMessagePtr message) {
if (!isLoggingInitialized()) { if (!isLoggingInitialized()) {
std::ostringstream error; std::ostringstream error;
@ -228,10 +230,10 @@ namespace g3 {
} }
/** The default, initial, handling to send a 'fatal' event to g3logworker /** The default, initial, handling to send a 'fatal' event to g3logworker
* the caller will stay here, eternally, until the software is aborted * the caller will stay here, eternally, until the software is aborted
* ... in the case of unit testing it is the given "Mock" fatalCall that will * ... in the case of unit testing it is the given "Mock" fatalCall that will
* define the behaviour. * define the behaviour.
*/ */
void fatalCall(FatalMessagePtr message) { void fatalCall(FatalMessagePtr message) {
g_fatal_to_g3logworker_function_ptr(FatalMessagePtr{std::move(message)}); g_fatal_to_g3logworker_function_ptr(FatalMessagePtr{std::move(message)});
} }

View File

@ -106,6 +106,8 @@ namespace g3 {
void saveMessage(const char* message, const char* file, int line, const char* function, const LEVELS& level, void saveMessage(const char* message, const char* file, int line, const char* function, const LEVELS& level,
const char* boolean_expression, int fatal_signal, const char* stack_trace); const char* boolean_expression, int fatal_signal, const char* stack_trace);
void saveFatalMessage(const LEVELS& level, const char* stack_trace, g3::LogMessagePtr& message, int& fatal_signal);
// forwards the message to all sinks // forwards the message to all sinks
void pushMessageToLogger(LogMessagePtr log_entry); void pushMessageToLogger(LogMessagePtr log_entry);