Start checking for a few possible resource exhaustion scenarios for explorer (#2793)

I couldn't figure out a way to actually hit a reasonable out-of-memory case once I add the maximum interpreter step count. However, the step count limit seems more important.

I've moved the todo stack limit out of function calls because there are plenty of ways to build up the todo stack without any function calls.

Fixes #2791

explorer/common/arena.h

@@ -35,6 +35,7 @@ class Arena {
         std::make_unique<ArenaEntryTyped<T>>(std::forward<Args>(args)...);
     Nonnull<T*> ptr = smart_ptr->Instance();
     arena_.push_back(std::move(smart_ptr));
+    allocated_ += sizeof(T);
     return ptr;
   }
 
@@ -45,8 +46,11 @@ class Arena {
   void New(WriteAddressTo<U> addr, Args&&... args) {
     arena_.push_back(std::make_unique<ArenaEntryTyped<T>>(
         addr, std::forward<Args>(args)...));
+    allocated_ += sizeof(T);
   }
 
+  auto allocated() -> int64_t { return allocated_; }
+
  private:
   // Virtualizes arena entries so that a single vector can contain many types,
   // avoiding templated statics.
@@ -83,6 +87,7 @@ class Arena {
 
   // Manages allocations in an arena for destruction at shutdown.
   std::vector<std::unique_ptr<ArenaEntry>> arena_;
+  int64_t allocated_ = 0;
 };
 
 }  // namespace Carbon

explorer/interpreter/action_stack.cpp

@@ -21,7 +21,7 @@ void ActionStack::Print(llvm::raw_ostream& out) const {
 
 void ActionStack::Start(std::unique_ptr<Action> action) {
   result_ = std::nullopt;
-  CARBON_CHECK(todo_.IsEmpty());
+  CARBON_CHECK(todo_.empty());
   todo_.Push(std::move(action));
 }
 
@@ -253,7 +253,7 @@ auto ActionStack::UnwindPast(Nonnull<const Statement*> ast_node,
 
 void ActionStack::PopScopes(
     std::stack<std::unique_ptr<Action>>& cleanup_stack) {
-  while (!todo_.IsEmpty() && llvm::isa<ScopeAction>(*todo_.Top())) {
+  while (!todo_.empty() && llvm::isa<ScopeAction>(*todo_.Top())) {
     auto act = todo_.Pop();
     if (act->scope()) {
       cleanup_stack.push(std::move(act));
@@ -262,7 +262,7 @@ void ActionStack::PopScopes(
 }
 
 void ActionStack::SetResult(Nonnull<const Value*> result) {
-  if (todo_.IsEmpty()) {
+  if (todo_.empty()) {
     result_ = result;
   } else {
     todo_.Top()->AddResult(result);

explorer/interpreter/action_stack.h

@@ -38,7 +38,7 @@ class ActionStack {
   void Start(std::unique_ptr<Action> action);
 
   // True if the stack is empty.
-  auto IsEmpty() const -> bool { return todo_.IsEmpty(); }
+  auto empty() const -> bool { return todo_.empty(); }
 
   // The Action currently at the top of the stack. This will never be a
   // ScopeAction.
@@ -106,7 +106,7 @@ class ActionStack {
 
   void Pop() { todo_.Pop(); }
 
-  auto Count() const -> int { return todo_.Count(); }
+  auto size() const -> int { return todo_.size(); }
 
  private:
   // Pop any ScopeActions from the top of the stack, propagating results as

explorer/interpreter/interpreter.cpp

@@ -40,6 +40,11 @@ using llvm::isa;
 
 namespace Carbon {
 
+// Limits for various overflow conditions.
+static constexpr int64_t MaxTodoSize = 1e3;
+static constexpr int64_t MaxStepsTaken = 1e6;
+static constexpr int64_t MaxArenaAllocated = 1e9;
+
 // Constructs an ActionStack suitable for the specified phase.
 static auto MakeTodo(Phase phase, Nonnull<Heap*> heap) -> ActionStack {
   switch (phase) {
@@ -185,6 +190,10 @@ class Interpreter {
   Nonnull<llvm::raw_ostream*> print_stream_;
 
   Phase phase_;
+
+  // The number of steps taken by the interpreter. Used for infinite loop
+  // detection.
+  int64_t steps_taken_ = 0;
 };
 
 //
@@ -951,10 +960,6 @@ auto Interpreter::CallFunction(const CallExpression& call,
                                Nonnull<const Value*> fun,
                                Nonnull<const Value*> arg,
                                ImplWitnessMap&& witnesses) -> ErrorOr<Success> {
-  constexpr int StackSizeLimit = 1000;
-  if (todo_.Count() > StackSizeLimit) {
-    return ProgramError(call.source_loc()) << "stack overflow";
-  }
   if (trace_stream_->is_enabled()) {
     *trace_stream_ << "calling function: " << *fun << "\n";
   }
@@ -2394,6 +2399,20 @@ auto Interpreter::StepCleanUp() -> ErrorOr<Success> {
 
 // State transition.
 auto Interpreter::Step() -> ErrorOr<Success> {
+  // Check for various overflow conditions before stepping.
+  if (todo_.size() > MaxTodoSize) {
+    return ProgramError(SourceLocation("overflow", 1))
+           << "Stack overflow: too many interpreter actions on stack";
+  }
+  if (++steps_taken_ > MaxStepsTaken) {
+    return ProgramError(SourceLocation("overflow", 1))
+           << "Possible infinite loop: too many interpreter steps executed";
+  }
+  if (arena_->allocated() > MaxArenaAllocated) {
+    return ProgramError(SourceLocation("overflow", 1))
+           << "Out of memory: exceeded arena allocation limit";
+  }
+
   Action& act = todo_.CurrentAction();
   switch (act.kind()) {
     case Action::Kind::LocationAction:
@@ -2434,7 +2453,7 @@ auto Interpreter::RunAllSteps(std::unique_ptr<Action> action)
     TraceState();
   }
   todo_.Start(std::move(action));
-  while (!todo_.IsEmpty()) {
+  while (!todo_.empty()) {
     CARBON_RETURN_IF_ERROR(Step());
     if (trace_stream_->is_enabled()) {
       TraceState();

explorer/interpreter/stack.h

@@ -32,7 +32,7 @@ struct Stack {
   //
   // - Requires: !this->IsEmpty()
   auto Pop() -> T {
-    CARBON_CHECK(!IsEmpty()) << "Can't pop from empty stack.";
+    CARBON_CHECK(!empty()) << "Can't pop from empty stack.";
     auto r = std::move(elements_.back());
     elements_.pop_back();
     return r;
@@ -52,15 +52,15 @@ struct Stack {
   //
   // - Requires: !this->IsEmpty()
   auto Top() const -> const T& {
-    CARBON_CHECK(!IsEmpty()) << "Empty stack has no Top().";
+    CARBON_CHECK(!empty()) << "Empty stack has no Top().";
     return elements_.back();
   }
 
   // Returns `true` iff `Count() > 0`.
-  auto IsEmpty() const -> bool { return elements_.empty(); }
+  auto empty() const -> bool { return elements_.empty(); }
 
   // Returns the number of elements in `*this`.
-  auto Count() const -> int { return elements_.size(); }
+  auto size() const -> int { return elements_.size(); }
 
   // Iterates over the Stack from top to bottom.
   auto begin() const -> const_iterator { return elements_.crbegin(); }

explorer/testdata/limits/README.md

@@ -0,0 +1,11 @@
+# Limit tests
+
+<!--
+Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+Exceptions. See /LICENSE for license information.
+SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+-->
+
+These tests check for various limit conditions (such as an infinite loop). The
+tests collectively disable autoupdate so that tracing isn't enabled, because
+tracing creates substantial additional overhead.

explorer/testdata/limits/fail_allocate.carbon

@@ -0,0 +1,20 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// NOAUTOUPDATE
+// RUN: %{not} %{explorer-run}
+// CHECK:STDERR: RUNTIME ERROR: overflow:1: Possible infinite loop: too many interpreter steps executed
+
+package EmptyIdentifier impl;
+
+fn Main() -> i32 {
+  while (true) {
+    // Ideally we would hit an OOM here, but it's too difficult to OOM from heap
+    // allocations before hitting max steps. Maybe with string operations we
+    // could trigger actual excessive memory allocations by just doubling the
+    // size of the string each time.
+    heap.New((0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0));
+  }
+  return 0;
+}

explorer/testdata/function/fail_recursion_stackoverflow.carbon → explorer/testdata/limits/fail_function_recursion.carbon

@@ -2,19 +2,17 @@
 // Exceptions. See /LICENSE for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// AUTOUPDATE
+// NOAUTOUPDATE
 // RUN: %{not} %{explorer-run}
-// RUN: %{not} %{explorer-run-trace}
+// CHECK:STDERR: RUNTIME ERROR: overflow:1: Stack overflow: too many interpreter actions on stack
 
 package EmptyIdentifier impl;
 
 fn A() {
-  // CHECK:STDERR: RUNTIME ERROR: {{.*}}/explorer/testdata/function/fail_recursion_stackoverflow.carbon:[[@LINE+1]]: stack overflow
   A();
 }
 
-fn Main() -> i32
-{
+fn Main() -> i32 {
   A();
   return 0;
 }

explorer/testdata/limits/fail_loop.carbon

@@ -0,0 +1,14 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// NOAUTOUPDATE
+// RUN: %{not} %{explorer-run}
+// CHECK:STDERR: RUNTIME ERROR: overflow:1: Possible infinite loop: too many interpreter steps executed
+
+package ExplorerTest impl;
+
+fn Main() -> i32 {
+  while (true) { }
+  return 0;
+}

explorer/testdata/limits/fail_type_check_loop.carbon

@@ -0,0 +1,18 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// NOAUTOUPDATE
+// RUN: %{not} %{explorer-run}
+// CHECK:STDERR: COMPILATION ERROR: overflow:1: Possible infinite loop: too many interpreter steps executed
+
+package ExplorerTest impl;
+
+fn Loop() -> type {
+  while (true) {}
+  return i32;
+}
+
+fn Main() -> Loop() {
+  return 0;
+}