Skip to content

Commit 714b75a

Browse files
mpirvumpirvu
authored
Merge pull request #19581 from gita-omr/v0.46.0-release
(0.46) Disclaim Cold Code Cache
2 parents e993860 + 29949c2 commit 714b75a

File tree

12 files changed

+285
-10
lines changed

12 files changed

+285
-10
lines changed

‎runtime/compiler/control/HookedByTheJit.cpp

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4610,10 +4610,22 @@ void disclaimIProfilerSegments(uint64_t crtElapsedTime)
46104610
}
46114611
}
46124612

4613+
void disclaimCodeCaches(uint64_t crtElapsedTime)
4614+
{
4615+
size_t rssBefore = getRSS_Kb();
4616+
int numDisclaimed = TR::CodeCacheManager::instance()->disclaimAllCodeCaches();
4617+
size_t rssAfter = getRSS_Kb();
4618+
if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
4619+
TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Code Caches RSS before=%zu KB, RSS after=%zu KB, delta=%zu KB = %5.2f%%",
4620+
(uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore));
4621+
}
4622+
46134623
void memoryDisclaimLogic(TR::CompilationInfo *compInfo, uint64_t crtElapsedTime, uint8_t jitState)
46144624
{
46154625
static uint64_t lastDataCacheDisclaimTime = 0;
46164626
static int32_t lastNumAllocatedDataCaches = 0;
4627+
static uint64_t lastCodeCacheDisclaimTime = 0;
4628+
static int32_t lastNumAllocatedCodeCaches = 0;
46174629
static uint64_t lastIProfilerDisclaimTime = 0;
46184630
static uint32_t lastNumCompilationsDuringIProfilerDisclaim = 0;
46194631

@@ -4644,6 +4656,25 @@ void memoryDisclaimLogic(TR::CompilationInfo *compInfo, uint64_t crtElapsedTime,
46444656
}
46454657
}
46464658
}
4659+
4660+
// Use logic similar to Data caches above
4661+
if (TR::CodeCacheManager::instance()->isDisclaimEnabled())
4662+
{
4663+
// Ensure we don't do it too often
4664+
if (crtElapsedTime > lastCodeCacheDisclaimTime + TR::Options::_minTimeBetweenMemoryDisclaims)
4665+
{
4666+
// Disclaim if at least one code cache has been allocated since the last disclaim
4667+
// or if there was a large time interval since the last disclaim
4668+
if (TR::CodeCacheManager::instance()->getCurrentNumberOfCodeCaches() > lastNumAllocatedCodeCaches ||
4669+
crtElapsedTime > lastCodeCacheDisclaimTime + 12 * TR::Options::_minTimeBetweenMemoryDisclaims)
4670+
{
4671+
disclaimCodeCaches(crtElapsedTime);
4672+
lastCodeCacheDisclaimTime = crtElapsedTime; // Update the time when disclaim was last performed
4673+
lastNumAllocatedCodeCaches = TR::CodeCacheManager::instance()->getCurrentNumberOfCodeCaches();
4674+
}
4675+
}
4676+
}
4677+
46474678
#if defined(J9VM_INTERP_PROFILING_BYTECODES)
46484679
if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableInterpreterProfiling))
46494680
{
@@ -5344,6 +5375,17 @@ static void jitStateLogic(J9JITConfig * jitConfig, TR::CompilationInfo * compInf
53445375
}
53455376
}
53465377

5378+
#ifdef DEBUG_CODE_DISCLAIM
5379+
static int printRSS = 0;
5380+
printRSS++;
5381+
if (printRSS == 4 && // ~every 2s
5382+
TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
5383+
{
5384+
TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "Current RSS %zuKB", getRSS_Kb());
5385+
printRSS = 0;
5386+
}
5387+
#endif
5388+
53475389
if (lateDisclaimNeeded)
53485390
{
53495391
CpuUtilization *cpuUtil = compInfo->getCpuUtil();

‎runtime/compiler/control/J9Options.cpp

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2736,6 +2736,7 @@ J9::Options::fePreProcess(void * base)
27362736
{
27372737
self()->setOption(TR_DisableDataCacheDisclaiming);
27382738
self()->setOption(TR_DisableIProfilerDataDisclaiming);
2739+
self()->setOption(TR_EnableCodeCacheDisclaiming, false);
27392740
}
27402741

27412742
return true;
@@ -2900,12 +2901,28 @@ J9::Options::fePostProcessJIT(void * base)
29002901
}
29012902

29022903
if (!self()->getOption(TR_DisableDataCacheDisclaiming) ||
2903-
!self()->getOption(TR_DisableIProfilerDataDisclaiming))
2904+
!self()->getOption(TR_DisableIProfilerDataDisclaiming) ||
2905+
self()->getOption(TR_EnableCodeCacheDisclaiming))
29042906
{
29052907
// Check requirements for memory disclaiming (Linux kernel and default page size)
29062908
TR::Options::disableMemoryDisclaimIfNeeded(jitConfig);
29072909
}
29082910

2911+
const char *ccOption = J9::Options::_externalOptionStrings[J9::ExternalOptions::Xcodecache];
2912+
J9JavaVM *vm = javaVM; // needed by FIND_ARG_IN_VMARGS macro
2913+
int32_t argIndex = FIND_ARG_IN_VMARGS(EXACT_MEMORY_MATCH, ccOption, 0);
2914+
2915+
if (argIndex >= 0)
2916+
{
2917+
if (jitConfig->codeCacheKB < 4*1024*1024)
2918+
self()->setOption(TR_EnableCodeCacheDisclaiming, false);
2919+
}
2920+
else if (TR::Compiler->target.isLinux() &&
2921+
self()->getOption(TR_EnableCodeCacheDisclaiming))
2922+
{
2923+
jitConfig->codeCacheKB *= 2;
2924+
}
2925+
29092926
#if defined(J9VM_OPT_JITSERVER)
29102927
self()->setupJITServerOptions();
29112928
#endif /* defined(J9VM_OPT_JITSERVER) */
@@ -2957,6 +2974,7 @@ J9::Options::disableMemoryDisclaimIfNeeded(J9JITConfig *jitConfig)
29572974
{
29582975
TR::Options::getCmdLineOptions()->setOption(TR_DisableDataCacheDisclaiming);
29592976
TR::Options::getCmdLineOptions()->setOption(TR_DisableIProfilerDataDisclaiming);
2977+
TR::Options::getCmdLineOptions()->setOption(TR_EnableCodeCacheDisclaiming, false);
29602978
}
29612979
return shouldDisableMemoryDisclaim;
29622980
}

‎runtime/compiler/control/OptionsPostRestore.cpp

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ J9::OptionsPostRestore::OptionsPostRestore(J9VMThread *vmThread, J9JITConfig *ji
9090
= J9::Options::_xrsSync
9191
|| options->getOption(TR_NoResumableTrapHandler)
9292
|| options->getOption(TR_DisableTraps);
93+
94+
_enableCodeCacheDisclaimingPreCheckpoint = options->getOption(TR_EnableCodeCacheDisclaiming);
9395
}
9496

9597
void
@@ -804,8 +806,19 @@ J9::OptionsPostRestore::postProcessInternalCompilerOptions()
804806
}
805807
}
806808

809+
810+
if (!_enableCodeCacheDisclaimingPreCheckpoint &&
811+
TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming))
812+
{
813+
TR::Options::getCmdLineOptions()->setOption(TR_EnableCodeCacheDisclaiming, false);
814+
815+
if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
816+
TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Code Cache disclaiming disabled since it was disabled before checkpoint");
817+
}
818+
807819
if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableDataCacheDisclaiming) ||
808-
!TR::Options::getCmdLineOptions()->getOption(TR_DisableIProfilerDataDisclaiming))
820+
!TR::Options::getCmdLineOptions()->getOption(TR_DisableIProfilerDataDisclaiming) ||
821+
TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming))
809822
{
810823
TR::Options::disableMemoryDisclaimIfNeeded(_jitConfig);
811824
}

‎runtime/compiler/control/OptionsPostRestore.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ class OptionsPostRestore
164164
bool _asyncCompilationPreCheckpoint;
165165
bool _disableTrapsPreCheckpoint;
166166
bool _disableAOTPostRestore;
167+
bool _enableCodeCacheDisclaimingPreCheckpoint;
167168

168169
int32_t _argIndexXjit;
169170
int32_t _argIndexXjitcolon;

‎runtime/compiler/env/J2IThunk.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ TR_MHJ2IThunk::allocate(
6262
else
6363
#endif /* defined(J9VM_OPT_JITSERVER) */
6464
{
65-
result = (TR_MHJ2IThunk*)cg->allocateCodeMemory(totalSize, true, false);
65+
bool disclaim = TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming);
66+
result = (TR_MHJ2IThunk*)cg->allocateCodeMemory(totalSize, !disclaim, false);
6667
}
6768
omrthread_jit_write_protect_disable();
6869
result->_codeSize = codeSize;

‎runtime/compiler/runtime/CRRuntime.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -654,9 +654,10 @@ TR::CRRuntime::prepareForCheckpoint()
654654
#endif
655655

656656
// Make sure the limit for the ghost files is at least as big as the data cache size
657-
if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableDataCacheDisclaiming))
657+
if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableDataCacheDisclaiming) ||
658+
TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming))
658659
{
659-
U_32 ghostFileLimit = vm->jitConfig->dataCacheKB * 1024; // convert to bytes
660+
U_32 ghostFileLimit = std::max(vm->jitConfig->dataCacheKB, vm->jitConfig->codeCacheTotalKB) * 1024; // convert to bytes
660661
vm->internalVMFunctions->setRequiredGhostFileLimit(vmThread, ghostFileLimit);
661662
}
662663

‎runtime/compiler/runtime/J9CodeCache.cpp

Lines changed: 106 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <algorithm>
2828
#include <stdlib.h>
2929
#include <string.h>
30+
#include <errno.h>
3031
#include "j9.h"
3132
#include "j9protos.h"
3233
#include "j9thread.h"
@@ -55,11 +56,21 @@
5556
#include "env/VerboseLog.hpp"
5657
#include "omrformatconsts.h"
5758

59+
// for madvise
60+
#ifdef LINUX
61+
#include <sys/mman.h>
62+
#ifndef MADV_NOHUGEPAGE
63+
#define MADV_NOHUGEPAGE 15
64+
#endif // MADV_NOHUGEPAGE
65+
#ifndef MADV_PAGEOUT
66+
#define MADV_PAGEOUT 21
67+
#endif // MADV_PAGEOUT
68+
#endif
69+
5870
OMR::CodeCacheMethodHeader *getCodeCacheMethodHeader(char *p, int searchLimit, J9JITExceptionTable * metaData);
5971

6072
#define addFreeBlock2(start, end) addFreeBlock2WithCallSite((start), (end), __FILE__, __LINE__)
6173

62-
6374
TR::CodeCache *
6475
J9::CodeCache::self()
6576
{
@@ -134,8 +145,48 @@ J9::CodeCache::initialize(TR::CodeCacheManager *manager,
134145

135146
if (!self()->OMR::CodeCache::initialize(manager, codeCacheSegment, allocatedCodeCacheSizeInBytes))
136147
return false;
148+
137149
self()->setInitialAllocationPointers();
138150

151+
#ifdef LINUX
152+
if (manager->isDisclaimEnabled())
153+
{
154+
J9JavaVM * javaVM = jitConfig->javaVM;
155+
PORT_ACCESS_FROM_JAVAVM(javaVM); // for j9vmem_supported_page_sizes
156+
157+
uint8_t *middle = _warmCodeAlloc + (_coldCodeAllocBase - _warmCodeAlloc) / 2;
158+
size_t round = j9vmem_supported_page_sizes()[0] - 1;
159+
middle = (uint8_t *)(((size_t)(middle + round)) & ~round);
160+
161+
TR_ASSERT_FATAL(_coldCodeAlloc > middle, "A code cache can't be smaller than a page");
162+
163+
size_t coldCacheSize = _coldCodeAlloc - middle;
164+
coldCacheSize = (coldCacheSize + round) & ~round;
165+
166+
if (madvise(middle, coldCacheSize, MADV_NOHUGEPAGE) != 0)
167+
{
168+
const char *error = strerror(errno);
169+
if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
170+
TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to set MADV_NOHUGEPAGE for code cache: %s: %p %zu", error, middle, coldCacheSize);
171+
}
172+
else if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
173+
{
174+
TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Forcing code cache cold region %p-%p to use default size memory pages", middle, middle + coldCacheSize);
175+
}
176+
177+
// If the memory segment is backed by a file, disable read-ahead
178+
// so that touching one byte brings a single page in
179+
if (codeCacheSegment->j9segment()->vmemIdentifier.allocator == OMRPORT_VMEM_RESERVE_USED_MMAP_SHM)
180+
{
181+
if (madvise(middle, coldCacheSize, MADV_RANDOM) != 0)
182+
{
183+
if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
184+
TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to set MADV_RANDOM for cold code cache");
185+
}
186+
}
187+
}
188+
#endif // ifdef LINUX
189+
139190
_manager->reportCodeLoadEvents();
140191

141192
return true;
@@ -763,3 +814,57 @@ extern "C"
763814
}
764815

765816
}
817+
818+
819+
int32_t
820+
J9::CodeCache::disclaim(TR::CodeCacheManager *manager, bool canDisclaimOnSwap)
821+
{
822+
int32_t disclaimDone = 0;
823+
824+
#ifdef LINUX
825+
J9JavaVM * javaVM = jitConfig->javaVM;
826+
PORT_ACCESS_FROM_JAVAVM(javaVM); // for j9vmem_supported_page_sizes
827+
828+
bool trace = TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance);
829+
uint8_t *disclaim_start = _coldCodeAlloc;
830+
size_t pageSize = j9vmem_supported_page_sizes()[0];
831+
size_t round = pageSize - 1;
832+
disclaim_start = (uint8_t *)(((size_t)(disclaim_start + round)) & ~round);
833+
834+
if (_coldCodeAllocBase <= disclaim_start)
835+
return 0;
836+
837+
size_t disclaim_size = (_coldCodeAllocBase - disclaim_start + round) & ~round;
838+
839+
if (trace)
840+
{
841+
size_t warm_size = _warmCodeAlloc - _segment->segmentBase() + sizeof(this);
842+
size_t cold_size = _coldCodeAllocBase - _coldCodeAlloc;
843+
844+
TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "Will disclaim cold code cache %p : coldStart=%p coldBase=%p warm_size=%zuB cold_size=%zuB cold_size/(cold_size + warm_size)=%5.2f%%\n",
845+
this, _coldCodeAlloc, _coldCodeAllocBase,
846+
warm_size, cold_size, cold_size * 100.0/(cold_size + warm_size));
847+
}
848+
849+
int32_t ret = madvise((void *)disclaim_start, disclaim_size, MADV_PAGEOUT);
850+
851+
if (ret != 0)
852+
{
853+
if (trace)
854+
TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Failed to use madvise to disclaim memory for code cache");
855+
856+
if (ret == EINVAL)
857+
{
858+
manager->setDisclaimEnabled(false); // Don't try to disclaim again, since support seems to be missing
859+
if (trace)
860+
TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Disabling data cache disclaiming from now on");
861+
}
862+
}
863+
else
864+
{
865+
disclaimDone = 1;
866+
}
867+
#endif // ifdef LINUX
868+
869+
return disclaimDone;
870+
}

‎runtime/compiler/runtime/J9CodeCache.hpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,8 @@ class OMR_EXTENSIBLE CodeCache : public OMR::CodeCacheConnector
103103
*/
104104
void resetCodeCache();
105105

106+
int32_t disclaim(TR::CodeCacheManager *manager, bool canDisclaimOnSwap);
107+
106108
private:
107109
/**
108110
* @brief Restore trampoline pointers to their initial positions

‎runtime/compiler/runtime/J9CodeCacheManager.cpp

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,19 @@ J9::CodeCacheManager::allocateCodeCacheSegment(size_t segmentSize,
427427
}
428428
#endif
429429

430+
#ifdef LINUX
431+
if (_disclaimEnabled)
432+
{
433+
// If swap is enabled, we can allocate memory with mmap(MAP_ANOYNMOUS|MAP_PRIVATE) and disclaim to swap
434+
// If swap is not enabled we can disclaim to a backing file
435+
TR::CompilationInfo * compInfo = TR::CompilationInfo::get(_jitConfig);
436+
if (!TR::Options::getCmdLineOptions()->getOption(TR_DisclaimMemoryOnSwap) || compInfo->isSwapMemoryDisabled())
437+
{
438+
segmentType |= MEMORY_TYPE_DISCLAIMABLE_TO_FILE;
439+
}
440+
}
441+
#endif
442+
430443
mcc_printf("TR::CodeCache::allocate : requesting %d bytes\n", codeCacheSizeToAllocate);
431444
mcc_printf("TR::CodeCache::allocate : javaVM = %p\n", javaVM);
432445
mcc_printf("TR::CodeCache::allocate : codeCacheList = %p\n", jitConfig->codeCacheList);
@@ -487,6 +500,7 @@ J9::CodeCacheManager::allocateCodeCacheSegment(size_t segmentSize,
487500
}
488501
TR_VerboseLog::writeLineLocked(TR_Vlog_CODECACHE, verboseLogString, codeCacheSegment->baseAddress, codeCacheSegment->heapTop, alignment, largeCodePageSize);
489502
}
503+
490504
#ifdef LINUX
491505
if (0 != madvise((void *)codeCacheSegment->baseAddress, (codeCacheSegment->heapTop - codeCacheSegment->baseAddress), MADV_HUGEPAGE))
492506
{
@@ -496,11 +510,21 @@ J9::CodeCacheManager::allocateCodeCacheSegment(size_t segmentSize,
496510
}
497511
}
498512
#endif // LINUX
513+
514+
if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
515+
TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Allocated new code cache segment %p starting at address %p %d Kb",
516+
codeCacheSegment,
517+
codeCacheSegment->heapBase,
518+
_jitConfig->codeCacheKB);
499519
}
500520
else
501521
{
502522
// TODO: we should generate a trace point
503523
mcc_printf("TR::CodeCache::allocate : codeCacheSegment is NULL, %p\n",codeCacheSegment);
524+
525+
if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
526+
TR_VerboseLog::writeLineLocked(TR_Vlog_INFO, "Failed to allocate new code cache segment of %d Kb", _jitConfig->codeCacheKB);
527+
504528
return 0;
505529
}
506530

@@ -788,3 +812,26 @@ J9::CodeCacheManager::printOccupancyStats()
788812
codeCache->printOccupancyStats();
789813
}
790814
}
815+
816+
817+
int32_t
818+
J9::CodeCacheManager::disclaimAllCodeCaches()
819+
{
820+
if (!_disclaimEnabled)
821+
return 0;
822+
823+
int32_t numDisclaimed = 0;
824+
825+
#ifdef LINUX
826+
TR::CompilationInfo *compInfo = TR::CompilationInfo::get(_jitConfig);
827+
bool canDisclaimOnSwap = TR::Options::getCmdLineOptions()->getOption(TR_DisclaimMemoryOnSwap) && !compInfo->isSwapMemoryDisabled();
828+
829+
CacheListCriticalSection scanCacheList(self());
830+
for (TR::CodeCache *codeCache = self()->getFirstCodeCache(); codeCache; codeCache = codeCache->next())
831+
{
832+
numDisclaimed += codeCache->disclaim(self(), canDisclaimOnSwap);
833+
}
834+
#endif // LINUX
835+
836+
return numDisclaimed;
837+
}

0 commit comments

Comments
 (0)