Generate error reports when a fuzz target exits.

Summary:
Implements https://github.com/google/sanitizers/issues/835.

Flush stdout before exiting in test cases.

Since the atexit hook is used for exit reports, pending prints to
stdout can be lost if they aren't flushed before calling exit().

Expect tests to have non-zero exit code if exit() is called.

Reviewers: vitalybuka, kcc

Reviewed By: kcc

Subscribers: eraman, llvm-commits, hiraditya

Differential Revision: https://reviews.llvm.org/D35602

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@308669 91177308-0d34-0410-b5e6-96231b3b80d8

21 files changed, 59 insertions, 20 deletions

diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp
index fd8cab38a7b..36d9422bc28 100644
--- a/lib/Fuzzer/FuzzerDriver.cpp
+++ b/lib/Fuzzer/FuzzerDriver.cpp
@@ -20,6 +20,7 @@
 #include <algorithm>
 #include <atomic>
 #include <chrono>
+#include <cstdlib>
 #include <cstring>
 #include <mutex>
 #include <string>
@@ -642,6 +643,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.HandleXfsz = Flags.handle_xfsz;
   SetSignalHandler(Options);
 
+  std::atexit(Fuzzer::StaticExitCallback);
+
   if (Flags.minimize_crash)
     return MinimizeCrashInput(Args, Options);
 
diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h
index 3fc3fe004ce..8602818a420 100644
--- a/lib/Fuzzer/FuzzerInternal.h
+++ b/lib/Fuzzer/FuzzerInternal.h
@@ -60,6 +60,7 @@ public:
 
   static void StaticAlarmCallback();
   static void StaticCrashSignalCallback();
+  static void StaticExitCallback();
   static void StaticInterruptCallback();
   static void StaticFileSizeExceedCallback();
 
@@ -91,6 +92,7 @@ public:
 private:
   void AlarmCallback();
   void CrashCallback();
+  void ExitCallback();
   void CrashOnOverwrittenData();
   void InterruptCallback();
   void MutateAndTestOne();
diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp
index ba4ba80db00..b9e70b6dadd 100644
--- a/lib/Fuzzer/FuzzerLoop.cpp
+++ b/lib/Fuzzer/FuzzerLoop.cpp
@@ -175,6 +175,11 @@ void Fuzzer::StaticCrashSignalCallback() {
   F->CrashCallback();
 }
 
+void Fuzzer::StaticExitCallback() {
+  assert(F);
+  F->ExitCallback();
+}
+
 void Fuzzer::StaticInterruptCallback() {
   assert(F);
   F->InterruptCallback();
@@ -198,6 +203,19 @@ void Fuzzer::CrashCallback() {
   _Exit(Options.ErrorExitCode);  // Stop right now.
 }
 
+void Fuzzer::ExitCallback() {
+  if (!RunningCB)
+    return; // This exit did not come from the user callback
+  Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid());
+  if (EF->__sanitizer_print_stack_trace)
+    EF->__sanitizer_print_stack_trace();
+  Printf("SUMMARY: libFuzzer: fuzz target exited\n");
+  DumpCurrentUnit("crash-");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode);
+}
+
+
 void Fuzzer::InterruptCallback() {
   Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
   PrintFinalStats();
diff --git a/lib/Fuzzer/test/AbsNegAndConstant64Test.cpp b/lib/Fuzzer/test/AbsNegAndConstant64Test.cpp
index dfb6007b797..abeb784e9a1 100644
--- a/lib/Fuzzer/test/AbsNegAndConstant64Test.cpp
+++ b/lib/Fuzzer/test/AbsNegAndConstant64Test.cpp
@@ -16,6 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   memcpy(&y, Data + sizeof(x), sizeof(y));
   if (llabs(x) < 0 && y == 0xbaddcafedeadbeefULL) {
     printf("BINGO; Found the target, exiting; x = 0x%lx y 0x%lx\n", x, y);
+    fflush(stdout);
     exit(1);
   }
   return 0;
diff --git a/lib/Fuzzer/test/AbsNegAndConstantTest.cpp b/lib/Fuzzer/test/AbsNegAndConstantTest.cpp
index e9d983ff1eb..049db0a60c3 100644
--- a/lib/Fuzzer/test/AbsNegAndConstantTest.cpp
+++ b/lib/Fuzzer/test/AbsNegAndConstantTest.cpp
@@ -16,6 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   memcpy(&y, Data + sizeof(x), sizeof(y));
   if (abs(x) < 0 && y == 0xbaddcafe) {
     printf("BINGO; Found the target, exiting; x = 0x%x y 0x%x\n", x, y);
+    fflush(stdout);
     exit(1);
   }
   return 0;
diff --git a/lib/Fuzzer/test/BufferOverflowOnInput.cpp b/lib/Fuzzer/test/BufferOverflowOnInput.cpp
index 75e1fb90a19..159da92d4e9 100644
--- a/lib/Fuzzer/test/BufferOverflowOnInput.cpp
+++ b/lib/Fuzzer/test/BufferOverflowOnInput.cpp
@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 
 static volatile bool SeedLargeBuffer;
 
@@ -15,7 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Size >= 4)
     SeedLargeBuffer = true;
   if (Size == 3 && SeedLargeBuffer && Data[3]) {
-    std::cout << "Woops, reading Data[3] w/o crashing\n";
+    std::cout << "Woops, reading Data[3] w/o crashing\n" << std::flush;
     exit(1);
   }
   return 0;
diff --git a/lib/Fuzzer/test/CustomCrossOverTest.cpp b/lib/Fuzzer/test/CustomCrossOverTest.cpp
index b624088b902..58059c9c1eb 100644
--- a/lib/Fuzzer/test/CustomCrossOverTest.cpp
+++ b/lib/Fuzzer/test/CustomCrossOverTest.cpp
@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 #include <random>
 #include <string.h>
 
@@ -26,7 +27,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Size && Data[0] == 'a') sink--;
 
   if (Str.find(Target) != std::string::npos) {
-    std::cout << "BINGO; Found the target, exiting\n";
+    std::cout << "BINGO; Found the target, exiting\n" << std::flush;
     exit(1);
   }
   return 0;
diff --git a/lib/Fuzzer/test/CustomMutatorTest.cpp b/lib/Fuzzer/test/CustomMutatorTest.cpp
index 521d7f506b4..b2adb94082d 100644
--- a/lib/Fuzzer/test/CustomMutatorTest.cpp
+++ b/lib/Fuzzer/test/CustomMutatorTest.cpp
@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 
 #include "FuzzerInterface.h"
 
@@ -19,7 +20,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     if (Size > 1 && Data[1] == 'i') {
       Sink = 2;
       if (Size > 2 && Data[2] == '!') {
-        std::cout << "BINGO; Found the target, exiting\n";
+        std::cout << "BINGO; Found the target, exiting\n" << std::flush;
         exit(1);
       }
     }
diff --git a/lib/Fuzzer/test/NthRunCrashTest.cpp b/lib/Fuzzer/test/NthRunCrashTest.cpp
index da5fbd33e96..26cdc8f17ad 100644
--- a/lib/Fuzzer/test/NthRunCrashTest.cpp
+++ b/lib/Fuzzer/test/NthRunCrashTest.cpp
@@ -5,12 +5,13 @@
 #include <cstddef>
 #include <cstdint>
 #include <iostream>
+#include <ostream>
 
 static int Counter;
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   if (Counter++ == 1000) {
-    std::cout << "BINGO; Found the target, exiting\n";
+    std::cout << "BINGO; Found the target, exiting\n" << std::flush;
     exit(1);
   }
   return 0;
diff --git a/lib/Fuzzer/test/RepeatedBytesTest.cpp b/lib/Fuzzer/test/RepeatedBytesTest.cpp
index 14222f28474..31868cf8c54 100644
--- a/lib/Fuzzer/test/RepeatedBytesTest.cpp
+++ b/lib/Fuzzer/test/RepeatedBytesTest.cpp
@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   assert(Data);
@@ -21,7 +22,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     MaxA = GT * CurA + (!GT) * MaxA;
   }
   if (MaxA >= 20) {
-    std::cout << "BINGO; Found the target (Max: " << MaxA << "), exiting\n";
+    std::cout << "BINGO; Found the target (Max: " << MaxA << "), exiting\n"
+              << std::flush;
     exit(0);
   }
   return 0;
diff --git a/lib/Fuzzer/test/SimpleDictionaryTest.cpp b/lib/Fuzzer/test/SimpleDictionaryTest.cpp
index a1cd2004722..ffa2e4137bd 100644
--- a/lib/Fuzzer/test/SimpleDictionaryTest.cpp
+++ b/lib/Fuzzer/test/SimpleDictionaryTest.cpp
@@ -10,6 +10,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
+#include <ostream>
 
 static volatile int Zero = 0;
 
@@ -21,7 +22,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     if (Expected[i] + Zero == Data[i])
       Match++;
   if (Match == strlen(Expected)) {
-    std::cout << "BINGO; Found the target, exiting\n";
+    std::cout << "BINGO; Found the target, exiting\n" << std::flush;
     exit(1);
   }
   return 0;
diff --git a/lib/Fuzzer/test/SimpleTest.cpp b/lib/Fuzzer/test/SimpleTest.cpp
index a8b4988dff1..3882a842b88 100644
--- a/lib/Fuzzer/test/SimpleTest.cpp
+++ b/lib/Fuzzer/test/SimpleTest.cpp
@@ -7,6 +7,7 @@
 #include <cstdint>
 #include <cstdlib>
 #include <iostream>
+#include <ostream>
 
 static volatile int Sink;
 
@@ -17,7 +18,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
     if (Size > 1 && Data[1] == 'i') {
       Sink = 2;
       if (Size > 2 && Data[2] == '!') {
-        std::cout << "BINGO; Found the target, exiting\n";
+        std::cout << "BINGO; Found the target, exiting\n" << std::flush;
         exit(0);
       }
     }
diff --git a/lib/Fuzzer/test/SimpleThreadedTest.cpp b/lib/Fuzzer/test/SimpleThreadedTest.cpp
index 1abdc3fc6d6..deeae756a82 100644
--- a/lib/Fuzzer/test/SimpleThreadedTest.cpp
+++ b/lib/Fuzzer/test/SimpleThreadedTest.cpp
@@ -7,12 +7,13 @@
 #include <cstdint>
 #include <cstring>
 #include <iostream>
+#include <ostream>
 #include <thread>
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
   auto C = [&] {
     if (Size >= 2 && Data[0] == 'H') {
-        std::cout << "BINGO; Found the target, exiting\n";
+        std::cout << "BINGO; Found the target, exiting\n" << std::flush;
         abort();
     }
   };
diff --git a/lib/Fuzzer/test/exit-report.test b/lib/Fuzzer/test/exit-report.test
new file mode 100644
index 00000000000..26e8cfc9155
--- /dev/null
+++ b/lib/Fuzzer/test/exit-report.test
@@ -0,0 +1,5 @@
+RUN: not LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s
+
+CHECK: ERROR: libFuzzer: fuzz target exited
+CHECK: SUMMARY: libFuzzer: fuzz target exited
+CHECK: Test unit written to
diff --git a/lib/Fuzzer/test/fuzzer-flags.test b/lib/Fuzzer/test/fuzzer-flags.test
index 61ce07797b4..b9bf2f8e4cc 100644
--- a/lib/Fuzzer/test/fuzzer-flags.test
+++ b/lib/Fuzzer/test/fuzzer-flags.test
@@ -1,18 +1,18 @@
-RUN: LLVMFuzzer-FlagsTest -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR
+RUN: not LLVMFuzzer-FlagsTest -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR
 FOO_BAR: WARNING: unrecognized flag '-foo_bar=1'; use -help=1 to list all flags
 FOO_BAR: BINGO
 
-RUN: LLVMFuzzer-FlagsTest -runs=10 --max_len=100 2>&1 | FileCheck %s --check-prefix=DASH_DASH
+RUN: not LLVMFuzzer-FlagsTest -runs=10 --max_len=100 2>&1 | FileCheck %s --check-prefix=DASH_DASH
 DASH_DASH: WARNING: did you mean '-max_len=100' (single dash)?
 DASH_DASH: INFO: A corpus is not provided, starting from an empty corpus
 
 RUN: LLVMFuzzer-FlagsTest -help=1 2>&1 | FileCheck %s --check-prefix=NO_INTERNAL
 NO_INTERNAL-NOT: internal flag
 
-RUN: LLVMFuzzer-FlagsTest --foo-bar -runs=10 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU
+RUN: not LLVMFuzzer-FlagsTest --foo-bar -runs=10 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU
 PASSTHRU: BINGO --foo-bar --baz -help=1 test
 
 RUN: mkdir -p %t/T0 %t/T1
 RUN: touch %t/T1/empty
-RUN: LLVMFuzzer-FlagsTest --foo-bar -merge=1 %t/T0 %t/T1 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU-MERGE
+RUN: not LLVMFuzzer-FlagsTest --foo-bar -merge=1 %t/T0 %t/T1 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU-MERGE
 PASSTHRU-MERGE: BINGO --foo-bar --baz -help=1 test
diff --git a/lib/Fuzzer/test/fuzzer-printcovpcs.test b/lib/Fuzzer/test/fuzzer-printcovpcs.test
index e4c6f0ed1df..77a2e7d147c 100644
--- a/lib/Fuzzer/test/fuzzer-printcovpcs.test
+++ b/lib/Fuzzer/test/fuzzer-printcovpcs.test
@@ -1,4 +1,4 @@
-RUN: LLVMFuzzer-SimpleTest -print_pcs=1 -seed=1 2>&1 | FileCheck %s --check-prefix=PCS
+RUN: not LLVMFuzzer-SimpleTest -print_pcs=1 -seed=1 2>&1 | FileCheck %s --check-prefix=PCS
 PCS-NOT: NEW_PC
 PCS:INITED
 PCS:NEW_PC: {{0x[a-f0-9]+}}
diff --git a/lib/Fuzzer/test/fuzzer.test b/lib/Fuzzer/test/fuzzer.test
index ff46d32b387..a3b7d93603f 100644
--- a/lib/Fuzzer/test/fuzzer.test
+++ b/lib/Fuzzer/test/fuzzer.test
@@ -1,10 +1,10 @@
 CHECK: BINGO
 Done1000000: Done 1000000 runs in
 
-RUN: LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s
 
 # only_ascii mode. Will perform some minimal self-validation.
-RUN: LLVMFuzzer-SimpleTest -only_ascii=1 2>&1
+RUN: not LLVMFuzzer-SimpleTest -only_ascii=1 2>&1
 
 RUN: LLVMFuzzer-SimpleCmpTest -max_total_time=1 -use_cmp=0 2>&1 | FileCheck %s --check-prefix=MaxTotalTime
 MaxTotalTime: Done {{.*}} runs in {{.}} second(s)
@@ -47,7 +47,7 @@ RUN: not LLVMFuzzer-DSOTest 2>&1 | FileCheck %s --check-prefix=DSO
 DSO: INFO: Loaded 3 modules
 DSO: BINGO
 
-RUN: LLVMFuzzer-SimpleTest  -exit_on_src_pos=SimpleTest.cpp:17                 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS
+RUN: LLVMFuzzer-SimpleTest  -exit_on_src_pos=SimpleTest.cpp:18                 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS
 RUN: LLVMFuzzer-ShrinkControlFlowTest  -exit_on_src_pos=ShrinkControlFlowTest.cpp:23 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS
 EXIT_ON_SRC_POS: INFO: found line matching '{{.*}}', exiting.
 
diff --git a/lib/Fuzzer/test/inline-8bit-counters.test b/lib/Fuzzer/test/inline-8bit-counters.test
index 8747af81451..ff84dd119c1 100644
--- a/lib/Fuzzer/test/inline-8bit-counters.test
+++ b/lib/Fuzzer/test/inline-8bit-counters.test
@@ -1,4 +1,4 @@
 REQUIRES: linux
 CHECK: INFO: Loaded 1 modules with {{.*}} inline 8-bit counters
 CHECK: BINGO
-RUN: LLVMFuzzer-SimpleTest-Inline8bitCounters -runs=1000000 -seed=1 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-SimpleTest-Inline8bitCounters -runs=1000000 -seed=1 2>&1 | FileCheck %s
diff --git a/lib/Fuzzer/test/repeated-bytes.test b/lib/Fuzzer/test/repeated-bytes.test
index 71394087939..fc72142f15d 100644
--- a/lib/Fuzzer/test/repeated-bytes.test
+++ b/lib/Fuzzer/test/repeated-bytes.test
@@ -1,2 +1,2 @@
 CHECK: BINGO
-RUN: LLVMFuzzer-RepeatedBytesTest -seed=1 -runs=1000000 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-RepeatedBytesTest -seed=1 -runs=1000000 2>&1 | FileCheck %s
diff --git a/lib/Fuzzer/test/trace-pc.test b/lib/Fuzzer/test/trace-pc.test
index 3709677b71b..93bb2c06154 100644
--- a/lib/Fuzzer/test/trace-pc.test
+++ b/lib/Fuzzer/test/trace-pc.test
@@ -1,2 +1,2 @@
 CHECK: BINGO
-RUN: LLVMFuzzer-SimpleTest-TracePC -runs=100000 -seed=1 2>&1 | FileCheck %s
+RUN: not LLVMFuzzer-SimpleTest-TracePC -runs=100000 -seed=1 2>&1 | FileCheck %s
diff --git a/lib/Fuzzer/test/ulimit.test b/lib/Fuzzer/test/ulimit.test
index c2faca13f72..ca4e5ffce15 100644
--- a/lib/Fuzzer/test/ulimit.test
+++ b/lib/Fuzzer/test/ulimit.test
@@ -1,4 +1,4 @@
 REQUIRES: posix
 
 RUN: ulimit -s 1000
-RUN: LLVMFuzzer-SimpleTest
+RUN: not LLVMFuzzer-SimpleTest