[heap] Fix recursive GCs caused by adjusting externally allocated memory

[email protected]
BUG=chromium:526244
LOG=N

Review URL: https://codereview.chromium.org/1325643002

Cr-Commit-Position: refs/heads/master@{#30478}

Author: mlippautz

src/api.cc

@@ -6873,6 +6873,7 @@ Local<Integer> v8::Integer::NewFromUnsigned(Isolate* isolate, uint32_t value) {
 
 void Isolate::CollectAllGarbage(const char* gc_reason) {
   i::Heap* heap = reinterpret_cast<i::Isolate*>(this)->heap();
+  DCHECK_EQ(heap->gc_state(), i::Heap::NOT_IN_GC);
   if (heap->incremental_marking()->IsStopped()) {
     if (heap->incremental_marking()->CanBeActivated()) {
       heap->StartIncrementalMarking(

src/heap/heap.cc

@@ -1800,6 +1800,9 @@ void Heap::RegisterNewArrayBuffer(bool in_new_space, void* data,
     RegisterNewArrayBufferHelper(live_array_buffers_for_scavenge_, data,
                                  length);
   }
+
+  // We may go over the limit of externally allocated memory here. We call the
+  // api function to trigger a GC in this case.
   reinterpret_cast<v8::Isolate*>(isolate_)
       ->AdjustAmountOfExternalAllocatedMemory(length);
 }
@@ -1842,16 +1845,13 @@ void Heap::FreeDeadArrayBuffers(bool from_scavenge) {
       live_array_buffers_for_scavenge_.erase(buffer.first);
     }
   }
-  size_t freed_memory = FreeDeadArrayBuffersHelper(
+
+  // Do not call through the api as this code is triggered while doing a GC.
+  amount_of_external_allocated_memory_ += FreeDeadArrayBuffersHelper(
       isolate_,
       from_scavenge ? live_array_buffers_for_scavenge_ : live_array_buffers_,
       from_scavenge ? not_yet_discovered_array_buffers_for_scavenge_
                     : not_yet_discovered_array_buffers_);
-  if (freed_memory) {
-    reinterpret_cast<v8::Isolate*>(isolate_)
-        ->AdjustAmountOfExternalAllocatedMemory(
-            -static_cast<int64_t>(freed_memory));
-  }
 }