[scudo] Soft and hard RSS limit checks

Summary:
This implements an opportunistic check for the RSS limit.

For ASan, this was implemented thanks to a background thread checking the
current RSS vs the set limit every 100ms. This was deemed problematic for Scudo
due to potential Android concerns (Zygote as pointed out by Aleksey) as well as
the general inconvenience of having a permanent background thread.

If a limit (soft or hard) is specified, we will attempt to update the RSS limit
status (exceeded or not) every 100ms. This is done in an opportunistic way: if
we can update it, we do it, if not we return the current status, mostly because
we don't need it to be fully consistent (it's done every 100ms anyway). If the
limit is exceeded `allocate` will act as if OOM for a soft limit, or just die
for a hard limit.

We use the `common_flags()`'s `hard_rss_limit_mb` & `soft_rss_limit_mb` for
configuration of the limits.

Reviewers: alekseyshl

Reviewed By: alekseyshl

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D40038

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@318301 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 106 insertions, 0 deletions

diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index 48aa9caf5..66870d581 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -226,6 +226,12 @@ struct ScudoAllocator {
   bool ZeroContents;
   bool DeleteSizeMismatch;
 
+  bool CheckRssLimit;
+  uptr HardRssLimitMb;
+  uptr SoftRssLimitMb;
+  atomic_uint8_t RssLimitExceeded;
+  atomic_uint64_t RssLastCheckedAtNS;
+
   explicit ScudoAllocator(LinkerInitialized)
     : AllocatorQuarantine(LINKER_INITIALIZED) {}
 
@@ -270,6 +276,8 @@ struct ScudoAllocator {
 
     SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
     BackendAllocator.init(common_flags()->allocator_release_to_os_interval_ms);
+    HardRssLimitMb = common_flags()->hard_rss_limit_mb;
+    SoftRssLimitMb = common_flags()->soft_rss_limit_mb;
     AllocatorQuarantine.Init(
         static_cast<uptr>(getFlags()->QuarantineSizeKb) << 10,
         static_cast<uptr>(getFlags()->ThreadLocalQuarantineSizeKb) << 10);
@@ -280,6 +288,10 @@ struct ScudoAllocator {
 
     GlobalPrng.init();
     Cookie = GlobalPrng.getU64();
+
+    CheckRssLimit = HardRssLimitMb || SoftRssLimitMb;
+    if (CheckRssLimit)
+      atomic_store_relaxed(&RssLastCheckedAtNS, NanoTime());
   }
 
   // Helper function that checks for a valid Scudo chunk. nullptr isn't.
@@ -293,6 +305,41 @@ struct ScudoAllocator {
     return getScudoChunk(UserBeg)->isValid();
   }
 
+  // Opportunistic RSS limit check. This will update the RSS limit status, if
+  // it can, every 100ms, otherwise it will just return the current one.
+  bool isRssLimitExceeded() {
+    u64 LastCheck = atomic_load_relaxed(&RssLastCheckedAtNS);
+    const u64 CurrentCheck = NanoTime();
+    if (LIKELY(CurrentCheck < LastCheck + (100ULL * 1000000ULL)))
+      return atomic_load_relaxed(&RssLimitExceeded);
+    if (!atomic_compare_exchange_weak(&RssLastCheckedAtNS, &LastCheck,
+                                      CurrentCheck, memory_order_relaxed))
+      return atomic_load_relaxed(&RssLimitExceeded);
+    // TODO(kostyak): We currently use sanitizer_common's GetRSS which reads the
+    //                RSS from /proc/self/statm by default. We might want to
+    //                call getrusage directly, even if it's less accurate.
+    const uptr CurrentRssMb = GetRSS() >> 20;
+    if (HardRssLimitMb && HardRssLimitMb < CurrentRssMb) {
+      Report("%s: hard RSS limit exhausted (%zdMb vs %zdMb)\n",
+             SanitizerToolName, HardRssLimitMb, CurrentRssMb);
+      DumpProcessMap();
+      Die();
+    }
+    if (SoftRssLimitMb) {
+      if (atomic_load_relaxed(&RssLimitExceeded)) {
+        if (CurrentRssMb <= SoftRssLimitMb)
+          atomic_store_relaxed(&RssLimitExceeded, false);
+      } else {
+        if (CurrentRssMb > SoftRssLimitMb) {
+          atomic_store_relaxed(&RssLimitExceeded, true);
+          Report("%s: soft RSS limit exhausted (%zdMb vs %zdMb)\n",
+                 SanitizerToolName, SoftRssLimitMb, CurrentRssMb);
+        }
+      }
+    }
+    return atomic_load_relaxed(&RssLimitExceeded);
+  }
+
   // Allocates a chunk.
   void *allocate(uptr Size, uptr Alignment, AllocType Type,
                  bool ForceZeroContents = false) {
@@ -312,6 +359,9 @@ struct ScudoAllocator {
     if (UNLIKELY(AlignedSize >= MaxAllowedMallocSize))
       return FailureHandler::OnBadRequest();
 
+    if (CheckRssLimit && UNLIKELY(isRssLimitExceeded()))
+        return FailureHandler::OnOOM();
+
     // Primary and Secondary backed allocations have a different treatment. We
     // deal with alignment requirements of Primary serviced allocations here,
     // but the Secondary will take care of its own alignment needs.
diff --git a/test/scudo/rss.c b/test/scudo/rss.c
new file mode 100644
index 000000000..4376290ad
--- /dev/null
+++ b/test/scudo/rss.c
@@ -0,0 +1,56 @@
+// RUN: %clang_scudo %s -o %t
+// RUN:                                                                                                  %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit
+// RUN: %env_scudo_opts="soft_rss_limit_mb=256"                                                          %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit
+// RUN: %env_scudo_opts="hard_rss_limit_mb=256"                                                          %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-nolimit
+// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=0"                           not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit
+// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=1"                               %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit-returnnull
+// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=0:can_use_proc_maps_statm=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit
+// RUN: %env_scudo_opts="soft_rss_limit_mb=64:allocator_may_return_null=1:can_use_proc_maps_statm=0"     %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-softlimit-returnnull
+// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=0"                           not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit
+// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=1"                           not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit
+// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=0:can_use_proc_maps_statm=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit
+// RUN: %env_scudo_opts="hard_rss_limit_mb=64:allocator_may_return_null=1:can_use_proc_maps_statm=0" not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-hardlimit
+
+// Tests that the soft and hard RSS limits work as intended. Without limit or
+// with a high limit, the test should pass without any malloc returning NULL or
+// the program dying.
+// If a limit is specified, it should return some NULL or die depending on
+// allocator_may_return_null. This should also work without statm.
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static const size_t kNumAllocs = 128;
+static const size_t kAllocSize = 1 << 20;  // 1MB.
+
+static void *allocs[kNumAllocs];
+
+int main(int argc, char *argv[]) {
+  int returned_null = 0;
+  for (int i = 0; i < kNumAllocs; i++) {
+    if ((i & 0xf) == 0)
+      usleep(50000);
+    allocs[i] = malloc(kAllocSize);
+    if (allocs[i])
+      memset(allocs[i], 0xff, kAllocSize);  // Dirty the pages.
+    else
+      returned_null++;
+  }
+  for (int i = 0; i < kNumAllocs; i++)
+    free(allocs[i]);
+  if (returned_null == 0)
+    printf("All malloc calls succeeded\n");
+  else
+    printf("%d malloc calls returned NULL\n", returned_null);
+  return 0;
+}
+
+// CHECK-nolimit: All malloc calls succeeded
+// CHECK-softlimit: soft RSS limit exhausted
+// CHECK-softlimit-NOT: malloc calls
+// CHECK-softlimit-returnnull: soft RSS limit exhausted
+// CHECK-softlimit-returnnull: malloc calls returned NULL
+// CHECK-hardlimit: hard RSS limit exhausted
+// CHECK-hardlimit-NOT: malloc calls