From f0884b5f7533a1613caf8f12cdb8d73e806710f7 Mon Sep 17 00:00:00 2001 From: MarioHewardt Date: Wed, 5 Jul 2023 13:39:52 -0700 Subject: [PATCH 1/8] Refactor procdump/profiler --- .devcontainer/devcontainer.json | 9 +++++ Makefile | 3 +- include/Includes.h | 2 ++ include/ProcDumpConfiguration.h | 11 ------ include/ProfilerCommon.h | 23 +++++++++++++ include/ProfilerHelpers.h | 4 +-- profiler/inc/ProcDumpProfiler.h | 1 + profiler/src/ProcDumpProfiler.cpp | 56 ++++++++++++++++++++----------- src/Monitor.c | 19 ++++++++++- src/ProfilerHelpers.c | 21 +++--------- 10 files changed, 98 insertions(+), 51 deletions(-) create mode 100644 include/ProfilerCommon.h diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 834790a..f2948c4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,5 +5,14 @@ "build": { // If you want to use a different docker file (such as Dockerfile_Rocky) you can specify it here "dockerfile": "Dockerfile_Ubuntu" + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cpptools-extension-pack", + "ms-vscode.makefile-tools" + ] + } } } diff --git a/Makefile b/Makefile index a464c5f..654a8bd 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ TESTOUT=$(BINDIR)/ProcDumpTestApplication # Profiler PROFSRCDIR=profiler/src PROFINCDIR=profiler/inc +PROFCOMMONINCDIR=include PROFCXXFLAGS ?= -DELPP_NO_DEFAULT_LOG_FILE -DELPP_THREAD_SAFE -g -pthread -shared --no-undefined -Wno-invalid-noreturn -Wno-pragma-pack -Wno-writable-strings -Wno-format-security -fPIC -fms-extensions -DHOST_64BIT -DBIT64 -DPAL_STDCPP_COMPAT -DPLATFORM_UNIX -std=c++11 PROFCLANG=clang++ @@ -54,7 +55,7 @@ install: cp procdump.1 $(DESTDIR)$(MANDIR) $(OBJDIR)/ProcDumpProfiler.so: $(PROFSRCDIR)/ClassFactory.cpp $(PROFSRCDIR)/ProcDumpProfiler.cpp $(PROFSRCDIR)/dllmain.cpp $(PROFSRCDIR)/corprof_i.cpp $(PROFSRCDIR)/easylogging++.cc | $(OBJDIR) - $(PROFCLANG) -o $@ $(PROFCXXFLAGS) -I $(PROFINCDIR) $^ + $(PROFCLANG) -o $@ $(PROFCXXFLAGS) -I $(PROFINCDIR) -I $(PROFCOMMONINCDIR) $^ ld -r -b binary -o $(OBJDIR)/ProcDumpProfiler.o $(OBJDIR)/ProcDumpProfiler.so $(OBJDIR)/%.o: $(SRCDIR)/%.c | $(OBJDIR) diff --git a/include/Includes.h b/include/Includes.h index 931af82..289e1dc 100644 --- a/include/Includes.h +++ b/include/Includes.h @@ -1,3 +1,4 @@ +#include "ProfilerCommon.h" #include "CoreDumpWriter.h" #include "Events.h" #include "GenHelpers.h" @@ -9,3 +10,4 @@ #include "Process.h" #include "DotnetHelpers.h" #include "ProfilerHelpers.h" + diff --git a/include/ProcDumpConfiguration.h b/include/ProcDumpConfiguration.h index eed347d..b62f22d 100644 --- a/include/ProcDumpConfiguration.h +++ b/include/ProcDumpConfiguration.h @@ -43,17 +43,6 @@ // Structs // ------------------- -enum TriggerType -{ - Processor, - Commit, - Timer, - Signal, - ThreadCount, - FileDescriptorCount, - Exception -}; - struct TriggerThread { pthread_t thread; diff --git a/include/ProfilerCommon.h b/include/ProfilerCommon.h new file mode 100644 index 0000000..bdf3c81 --- /dev/null +++ b/include/ProfilerCommon.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License + +//-------------------------------------------------------------------- +// +// Profiler Common +// +//-------------------------------------------------------------------- +#ifndef PROFILERCOMMON_H +#define PROFILERCOMMON_H + +enum TriggerType +{ + Processor, + Commit, + Timer, + Signal, + ThreadCount, + FileDescriptorCount, + Exception +}; + +#endif // PROFILERCOMMON_H \ No newline at end of file diff --git a/include/ProfilerHelpers.h b/include/ProfilerHelpers.h index 1c065c3..e300b2d 100644 --- a/include/ProfilerHelpers.h +++ b/include/ProfilerHelpers.h @@ -14,8 +14,8 @@ #define PROFILER_FILE_NAME "procdumpprofiler.so" #define PROFILER_GUID "{cf0d821e-299b-5307-a3d8-b283c03916dd}" -int InjectProfiler(pid_t pid, char* filter, char* fullDumpPath); -int LoadProfiler(pid_t pid, char* filter, char* fullDumpPath); +int InjectProfiler(pid_t pid, char* clientData); +int LoadProfiler(pid_t pid, char* clientData); int ExtractProfiler(); char* GetEncodedExceptionFilter(char* exceptionFilterCmdLine, unsigned int numDumps); diff --git a/profiler/inc/ProcDumpProfiler.h b/profiler/inc/ProcDumpProfiler.h index 9f05441..00a55f1 100644 --- a/profiler/inc/ProcDumpProfiler.h +++ b/profiler/inc/ProcDumpProfiler.h @@ -19,6 +19,7 @@ #include "cor.h" #include "corprof.h" #include "profilerstring.h" +#include "profilercommon.h" #include "easylogging++.h" #define DETACH_TIMEOUT 30000 diff --git a/profiler/src/ProcDumpProfiler.cpp b/profiler/src/ProcDumpProfiler.cpp index fffb205..cb315f2 100644 --- a/profiler/src/ProcDumpProfiler.cpp +++ b/profiler/src/ProcDumpProfiler.cpp @@ -310,9 +310,11 @@ WCHAR* CorProfiler::GetUint16(char* buffer) } //------------------------------------------------------------------------------------------------------------------------------------------------------ -// CorProfiler::ParseExceptionList +// CorProfiler::ParseClientData // -// Syntax of client data: ;;:;:,... +// Syntax of client data: ;... +// +// DOTNET_EXCEPTION_TRIGGER;;;:;:,... // //------------------------------------------------------------------------------------------------------------------------------------------------------ bool CorProfiler::ParseClientData(char* filter) @@ -321,6 +323,8 @@ bool CorProfiler::ParseClientData(char* filter) std::stringstream exceptionFilter(filter); std::string segment; std::vector exclist; + enum TriggerType trigger_type; + while(std::getline(exceptionFilter, segment, ';')) { @@ -331,10 +335,20 @@ bool CorProfiler::ParseClientData(char* filter) int i=0; for(std::string exception : exclist) { - if(i==0) + if(i == 0) { // - // First part of the exception list is either: + // First part of list is the type of trigger that is being invoked. + // + trigger_type = static_cast(std::stoi(exception)); + LOG(TRACE) << "CorProfiler::ParseClientData: trigger type = " << trigger_type; + i++; + continue; + } + if(i == 1) + { + // + // Second part of the list is either: // > base path to dump location (if it ends with '/') // > full path to dump file // @@ -344,10 +358,10 @@ bool CorProfiler::ParseClientData(char* filter) i++; continue; } - if(i==1) + if(i == 2) { // - // Second part of the exception list is always the procdump pid. + // Third part of the list is always the procdump pid. // we we need this to communicate back status to procdump // procDumpPid = std::stoi(exception); @@ -356,20 +370,22 @@ bool CorProfiler::ParseClientData(char* filter) continue; } - - // exception filter - std::string segment2; - std::stringstream stream(exception); - ExceptionMonitorEntry entry; - entry.exceptionID = NULL; - entry.collectedDumps = 0; - std::getline(stream, segment2, ':'); - - entry.exception = segment2; - std::getline(stream, segment2, ':'); - entry.dumpsToCollect = std::stoi(segment2); - - exceptionMonitorList.push_back(entry); + if(trigger_type == Exception) + { + // exception filter + std::string segment2; + std::stringstream stream(exception); + ExceptionMonitorEntry entry; + entry.exceptionID = NULL; + entry.collectedDumps = 0; + std::getline(stream, segment2, ':'); + + entry.exception = segment2; + std::getline(stream, segment2, ':'); + entry.dumpsToCollect = std::stoi(segment2); + + exceptionMonitorList.push_back(entry); + } } for (auto & element : exceptionMonitorList) diff --git a/src/Monitor.c b/src/Monitor.c index 9afd7b7..312ccd5 100644 --- a/src/Monitor.c +++ b/src/Monitor.c @@ -1242,8 +1242,12 @@ void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguratio auto_free char* exceptionFilter = NULL; auto_free char* fullDumpPath = NULL; auto_cancel_thread pthread_t waitForProfilerCompletion = -1; + auto_free char* clientData = NULL; int rc = 0; + unsigned int clientDataSize = 0; + + enum TriggerType type = Exception; if ((rc = WaitForQuitOrEvent(config, &config->evtStartMonitoring, INFINITE_WAIT)) == WAIT_OBJECT_0 + 1) { @@ -1323,7 +1327,20 @@ void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguratio pthread_mutex_unlock(&config->dotnetMutex); // Inject the profiler into the target process - if(InjectProfiler(config->ProcessId, exceptionFilter, fullDumpPath)!=0) + + // client data in the following format: + // exception_trigger;;;:;:,... + clientDataSize = snprintf(NULL, 0, "%d;%s;%d;%s", type, fullDumpPath, getpid(), exceptionFilter) + 1; + clientData = malloc(clientDataSize); + if(clientData==NULL) + { + Trace("ExceptionMonitoringThread: Failed to allocate memory for client data."); + return NULL; + } + + sprintf(clientData, "%d;%s;%d;%s", type, fullDumpPath, getpid(), exceptionFilter); + + if(InjectProfiler(config->ProcessId, clientData)!=0) { Trace("ExceptionMonitoring: Failed to inject the profiler."); pthread_cancel(waitForProfilerCompletion); diff --git a/src/ProfilerHelpers.c b/src/ProfilerHelpers.c index 6938f09..f0088cb 100644 --- a/src/ProfilerHelpers.c +++ b/src/ProfilerHelpers.c @@ -59,7 +59,7 @@ int ExtractProfiler() // process instructing the runtime to load the profiler. // //-------------------------------------------------------------------- -int LoadProfiler(pid_t pid, char* filter, char* fullDumpPath) +int LoadProfiler(pid_t pid, char* clientData) { struct sockaddr_un addr = {0}; uint32_t attachTimeout = 5000; @@ -68,7 +68,6 @@ int LoadProfiler(pid_t pid, char* filter, char* fullDumpPath) auto_free_fd int fd = -1; auto_free char* socketName = NULL; - auto_free char* clientData = NULL; auto_free char* dumpPath = NULL; auto_free uint16_t* profilerPathW = NULL; auto_free void* temp_buffer = NULL; @@ -119,19 +118,9 @@ int LoadProfiler(pid_t pid, char* filter, char* fullDumpPath) payloadSize += profilerPathLen*sizeof(uint16_t); // client data - if(filter) + if(clientData) { - // client data in the following format: - // ;;:;:,... - clientDataSize = snprintf(NULL, 0, "%s;%d;%s", fullDumpPath, getpid(), filter) + 1; - clientData = malloc(clientDataSize); - if(clientData==NULL) - { - Trace("LoadProfiler: Failed to allocate memory for client data."); - return -1; - } - - sprintf(clientData, "%s;%d;%s", fullDumpPath, getpid(), filter); + clientDataSize = strlen(clientData) + 1; } else { @@ -261,7 +250,7 @@ int LoadProfiler(pid_t pid, char* filter, char* fullDumpPath) // cancellation requests to. // //-------------------------------------------------------------------- -int InjectProfiler(pid_t pid, char* filter, char* fullDumpPath) +int InjectProfiler(pid_t pid, char* clientData) { int ret = ExtractProfiler(); if(ret != 0) @@ -271,7 +260,7 @@ int InjectProfiler(pid_t pid, char* filter, char* fullDumpPath) return ret; } - ret = LoadProfiler(pid, filter, fullDumpPath); + ret = LoadProfiler(pid, clientData); if(ret != 0) { Log(error, "Failed to load profiler. Please make sure you are running elevated and targetting a .NET process."); From 61617cb34a0dccebd444120b98a507f96a42a534 Mon Sep 17 00:00:00 2001 From: Mario Hewardt Date: Thu, 6 Jul 2023 07:12:33 -0700 Subject: [PATCH 2/8] initial checkin --- include/Monitor.h | 5 +- include/ProcDumpConfiguration.h | 2 + include/ProfilerCommon.h | 3 +- profiler/inc/ProcDumpProfiler.h | 5 +- profiler/src/ProcDumpProfiler.cpp | 154 ++++++++++++++---- src/Monitor.c | 199 +++++++++++++++++++----- src/ProcDumpConfiguration.c | 53 +++++-- tests/integration/TestWebApi/Program.cs | 6 + 8 files changed, 349 insertions(+), 78 deletions(-) diff --git a/include/Monitor.h b/include/Monitor.h index 77c845a..98efcfb 100644 --- a/include/Monitor.h +++ b/include/Monitor.h @@ -30,6 +30,9 @@ bool IsQuit(struct ProcDumpConfiguration *self); int SetQuit(struct ProcDumpConfiguration *self, int quit); bool ContinueMonitoring(struct ProcDumpConfiguration *self); bool BeginMonitoring(struct ProcDumpConfiguration *self); +bool MonitorDotNet(struct ProcDumpConfiguration *self); +char* GetThresholds(struct ProcDumpConfiguration *self); +char* GetClientData(struct ProcDumpConfiguration *self, char* fullDumpPath); // Monitor worker threads void *CommitMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */); @@ -38,7 +41,7 @@ void *ThreadCountMonitoringThread(void *thread_args /* struct ProcDumpConfigurat void *FileDescriptorCountMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */); void *SignalMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */); void *TimerThread(void *thread_args /* struct ProcDumpConfiguration* */); -void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */); +void *DotNetMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */); void *ProcessMonitor(void *thread_args /* struct ProcDumpConfiguration* */); void *WaitForProfilerCompletion(void *thread_args /* struct ProcDumpConfiguration* */); diff --git a/include/ProcDumpConfiguration.h b/include/ProcDumpConfiguration.h index b62f22d..cd3b04d 100644 --- a/include/ProcDumpConfiguration.h +++ b/include/ProcDumpConfiguration.h @@ -85,8 +85,10 @@ struct ProcDumpConfiguration { int CpuThreshold; // -c bool bCpuTriggerBelowValue; // -cl int* MemoryThreshold; // -m + int MemoryThresholdCount; int MemoryCurrentThreshold; bool bMemoryTriggerBelowValue; // -m or -ml + bool bMonitoringGCMemory; // -gcm int ThresholdSeconds; // -s bool bTimerThreshold; // -s int NumberOfDumpsToCollect; // -n diff --git a/include/ProfilerCommon.h b/include/ProfilerCommon.h index bdf3c81..5018552 100644 --- a/include/ProfilerCommon.h +++ b/include/ProfilerCommon.h @@ -17,7 +17,8 @@ enum TriggerType Signal, ThreadCount, FileDescriptorCount, - Exception + Exception, + GCThreshold }; #endif // PROFILERCOMMON_H \ No newline at end of file diff --git a/profiler/inc/ProcDumpProfiler.h b/profiler/inc/ProcDumpProfiler.h index 00a55f1..171a5d1 100644 --- a/profiler/inc/ProcDumpProfiler.h +++ b/profiler/inc/ProcDumpProfiler.h @@ -19,7 +19,7 @@ #include "cor.h" #include "corprof.h" #include "profilerstring.h" -#include "profilercommon.h" +#include "ProfilerCommon.h" #include "easylogging++.h" #define DETACH_TIMEOUT 30000 @@ -104,10 +104,12 @@ class CorProfiler : public ICorProfilerCallback8 ICorProfilerInfo* corProfilerInfo; ICorProfilerInfo8* corProfilerInfo8; std::vector exceptionMonitorList; + std::vector gcMemoryThresholdMonitorList; pthread_t healthThread; std::string processName; std::string fullDumpPath; pthread_mutex_t endDumpCondition; + enum TriggerType triggerType; String GetExceptionName(ObjectID objectId); String GetExceptionMessage(ObjectID objectId); @@ -123,6 +125,7 @@ class CorProfiler : public ICorProfilerCallback8 int send_all(int socket, void* buffer, size_t length); int recv_all(int socket, void* buffer, size_t length); bool WildcardSearch(WCHAR*, WCHAR*); + u_int64_t GetGCHeapSize(); public: CorProfiler(); diff --git a/profiler/src/ProcDumpProfiler.cpp b/profiler/src/ProcDumpProfiler.cpp index cb315f2..e116d0e 100644 --- a/profiler/src/ProcDumpProfiler.cpp +++ b/profiler/src/ProcDumpProfiler.cpp @@ -217,24 +217,24 @@ HRESULT STDMETHODCALLTYPE CorProfiler::InitializeForAttach(IUnknown *pCorProfile { LOG(TRACE) << "CorProfiler::InitializeForAttach: Enter"; - char* filter = new char[cbClientData+1]; - if(filter==NULL) + char* clientData = new char[cbClientData+1]; + if(clientData == NULL) { - LOG(TRACE) << "CorProfiler::InitializeForAttach: Failed to allocate memory for exception filter"; + LOG(TRACE) << "CorProfiler::InitializeForAttach: Failed to allocate memory for clientData filter"; return E_FAIL; } - memcpy(filter, pvClientData, cbClientData); - filter[cbClientData]='\0'; + memcpy(clientData, pvClientData, cbClientData); + clientData[cbClientData]='\0'; - if(ParseClientData(filter)==false) + if(ParseClientData(clientData)==false) { LOG(TRACE) << "CorProfiler::InitializeForAttach: Failed to parse client data"; - delete[] filter; + delete[] clientData; return E_FAIL; } - delete[] filter; + delete[] clientData; processName = GetProcessName(); if(processName.empty()) @@ -264,11 +264,27 @@ HRESULT STDMETHODCALLTYPE CorProfiler::InitializeForAttach(IUnknown *pCorProfile return E_FAIL; } - DWORD eventMask = COR_PRF_MONITOR_EXCEPTIONS; - HRESULT hr = this->corProfilerInfo8->SetEventMask(eventMask); - if(FAILED(hr)) + if(triggerType == Exception) { - LOG(TRACE) << "CorProfiler::InitializeForAttach: Failed to set event mask"; + HRESULT hr = this->corProfilerInfo8->SetEventMask(COR_PRF_MONITOR_EXCEPTIONS); + if(FAILED(hr)) + { + LOG(TRACE) << "CorProfiler::InitializeForAttach: Failed to set event mask: COR_PRF_MONITOR_EXCEPTIONS"; + return E_FAIL; + } + } + else if(triggerType == GCThreshold) + { + HRESULT hr = this->corProfilerInfo8->SetEventMask2(0, COR_PRF_HIGH_BASIC_GC); + if(FAILED(hr)) + { + LOG(TRACE) << "CorProfiler::InitializeForAttach: Failed to set event mask: COR_PRF_HIGH_BASIC_GC"; + return E_FAIL; + } + } + else + { + LOG(TRACE) << "CorProfiler::InitializeForAttach: Invalid trigger type found"; return E_FAIL; } @@ -323,8 +339,6 @@ bool CorProfiler::ParseClientData(char* filter) std::stringstream exceptionFilter(filter); std::string segment; std::vector exclist; - enum TriggerType trigger_type; - while(std::getline(exceptionFilter, segment, ';')) { @@ -340,8 +354,8 @@ bool CorProfiler::ParseClientData(char* filter) // // First part of list is the type of trigger that is being invoked. // - trigger_type = static_cast(std::stoi(exception)); - LOG(TRACE) << "CorProfiler::ParseClientData: trigger type = " << trigger_type; + triggerType = static_cast(std::stoi(exception)); + LOG(TRACE) << "CorProfiler::ParseClientData: trigger type = " << triggerType; i++; continue; } @@ -370,7 +384,7 @@ bool CorProfiler::ParseClientData(char* filter) continue; } - if(trigger_type == Exception) + if(triggerType == Exception) { // exception filter std::string segment2; @@ -386,6 +400,10 @@ bool CorProfiler::ParseClientData(char* filter) exceptionMonitorList.push_back(entry); } + else if (triggerType == GCThreshold) + { + gcMemoryThresholdMonitorList.push_back(std::stoi(exception)); + } } for (auto & element : exceptionMonitorList) @@ -393,6 +411,11 @@ bool CorProfiler::ParseClientData(char* filter) LOG(TRACE) << "CorProfiler::ParseClientData:Exception filter " << element.exception << " with dump count set to " << std::to_string(element.dumpsToCollect); } + for (auto & element : gcMemoryThresholdMonitorList) + { + LOG(TRACE) << "CorProfiler::ParseClientData:GCMemoryThreshold filter " << element; + } + LOG(TRACE) << "CorProfiler::ParseClientData: Exit"; return true; } @@ -665,6 +688,93 @@ std::string CorProfiler::GetDumpName(u_int16_t dumpCount,std::string exceptionNa return tmp.str(); } +//------------------------------------------------------------------------------------------------------------------------------------------------------ +// CorProfiler::GetGCHeapSize +//------------------------------------------------------------------------------------------------------------------------------------------------------ +uint64_t CorProfiler::GetGCHeapSize() +{ + LOG(TRACE) << "CorProfiler::GetGCHeapSize: Enter"; + uint64_t gcHeapSize = 0; + + ULONG nObjectRanges = 0; + bool fHeapAlloc = false; + COR_PRF_GC_GENERATION_RANGE* pObjectRanges = NULL; + const ULONG cRanges = 32; + COR_PRF_GC_GENERATION_RANGE objectRangesStackBuffer[cRanges]; + + HRESULT hr = corProfilerInfo8->GetGenerationBounds(cRanges, &nObjectRanges, objectRangesStackBuffer); + if (FAILED(hr)) + { + LOG(TRACE) << "CorProfiler::GetGCHeapSize: Failed calling GetGenerationBounds " << hr; + return E_FAIL; + } + + if (nObjectRanges <= cRanges) + { + pObjectRanges = objectRangesStackBuffer; + } + + if (pObjectRanges == NULL) + { + pObjectRanges = new COR_PRF_GC_GENERATION_RANGE[nObjectRanges]; + if (pObjectRanges == NULL) + { + LOG(TRACE) << "CorProfiler::GetGCHeapSize: Failed to allocate memory for object ranges "; + return E_FAIL; + } + + fHeapAlloc = true; + + ULONG nObjectRanges2 = 0; + HRESULT hr = corProfilerInfo8->GetGenerationBounds(nObjectRanges, &nObjectRanges2, pObjectRanges); + if (FAILED(hr) || nObjectRanges != nObjectRanges2) + { + LOG(TRACE) << "CorProfiler::GetGCHeapSize: Failed to call GetGenerationBounds with allocated memory"; + return E_FAIL; + } + } + + LOG(TRACE) << "#######################CorProfiler::GetGCHeapSize: nObjectRanges " << nObjectRanges; + + for (int i = nObjectRanges - 1; i >= 0; i--) + { + LOG(TRACE) << "Range " << i << " Length " << pObjectRanges[i].rangeLength; + gcHeapSize += pObjectRanges[i].rangeLength; + } + + return gcHeapSize; +} + +//------------------------------------------------------------------------------------------------------------------------------------------------------ +// CorProfiler::GarbageCollectionStarted +//------------------------------------------------------------------------------------------------------------------------------------------------------ +HRESULT STDMETHODCALLTYPE CorProfiler::GarbageCollectionStarted(int cGenerations, BOOL generationCollected[], COR_PRF_GC_REASON reason) +{ + LOG(TRACE) << "CorProfiler::GarbageCollectionStarted: Enter"; + + uint64_t size = GetGCHeapSize(); + + LOG(TRACE) << "CorProfiler::GarbageCollectionStarted: Total heap size " << size; + + LOG(TRACE) << "CorProfiler::GarbageCollectionStarted: Exit"; + return S_OK; +} + +//------------------------------------------------------------------------------------------------------------------------------------------------------ +// CorProfiler::GarbageCollectionFinished +//------------------------------------------------------------------------------------------------------------------------------------------------------ +HRESULT STDMETHODCALLTYPE CorProfiler::GarbageCollectionFinished() +{ + LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Enter"; + + uint64_t size = GetGCHeapSize(); + + LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Total heap size " << size; + + LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Exit"; + return S_OK; +} + //------------------------------------------------------------------------------------------------------------------------------------------------------ // CorProfiler::ExceptionThrown //------------------------------------------------------------------------------------------------------------------------------------------------------ @@ -1686,21 +1796,11 @@ HRESULT STDMETHODCALLTYPE CorProfiler::ThreadNameChanged(ThreadID threadId, ULON return S_OK; } -HRESULT STDMETHODCALLTYPE CorProfiler::GarbageCollectionStarted(int cGenerations, BOOL generationCollected[], COR_PRF_GC_REASON reason) -{ - return S_OK; -} - HRESULT STDMETHODCALLTYPE CorProfiler::SurvivingReferences(ULONG cSurvivingObjectIDRanges, ObjectID objectIDRangeStart[], ULONG cObjectIDRangeLength[]) { return S_OK; } -HRESULT STDMETHODCALLTYPE CorProfiler::GarbageCollectionFinished() -{ - return S_OK; -} - HRESULT STDMETHODCALLTYPE CorProfiler::FinalizeableObjectQueued(DWORD finalizerFlags, ObjectID objectID) { return S_OK; diff --git a/src/Monitor.c b/src/Monitor.c index 312ccd5..faa2a34 100644 --- a/src/Monitor.c +++ b/src/Monitor.c @@ -500,6 +500,20 @@ void MonitorProcesses(struct ProcDumpConfiguration *self) } } +//-------------------------------------------------------------------- +// +// MonitorDotNet - Returns true if we are monitoring a dotnet process +// +//-------------------------------------------------------------------- +bool MonitorDotNet(struct ProcDumpConfiguration *self) +{ + if(self->bDumpOnException || self->bMonitoringGCMemory) + { + return true; + } + + return false; +} //-------------------------------------------------------------------- // @@ -513,13 +527,13 @@ int CreateMonitorThreads(struct ProcDumpConfiguration *self) bool tooManyTriggers = false; // create threads - if (self->bDumpOnException) + if (MonitorDotNet(self) == true) { if (self->nThreads < MAX_TRIGGERS) { - if ((rc = pthread_create(&self->Threads[self->nThreads].thread, NULL, ExceptionMonitoringThread, (void *)self)) != 0) + if ((rc = pthread_create(&self->Threads[self->nThreads].thread, NULL, DotNetMonitoringThread, (void *)self)) != 0) { - Trace("CreateMonitorThreads: failed to create ExceptionMonitoringThread."); + Trace("CreateMonitorThreads: failed to create DotNetMonitoringThread."); return rc; } @@ -1231,42 +1245,36 @@ void *TimerThread(void *thread_args /* struct ProcDumpConfiguration* */) //-------------------------------------------------------------------- // -// ExceptionMonitoringThread - Thread that creates dumps based on -// exception filter. NOTE: .NET only. +// DotNetMonitoringThread - Thread that creates dumps based on +// dotnet triggers. +// +// NOTE: .NET only. +// NOTE: At the moment, .NET triggers are mutually exclusive meaning +// only one can specified at a time. For example, you cannot specify +// both exception based monitoring and GC based monitoring. // //-------------------------------------------------------------------- -void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */) +void *DotNetMonitoringThread(void *thread_args /* struct ProcDumpConfiguration* */) { - Trace("ExceptionMonitoring: Starting ExceptionMonitoring Thread"); + Trace("DotNetMonitoringThread: Starting DotNetMonitoringThread Thread"); struct ProcDumpConfiguration *config = (struct ProcDumpConfiguration *)thread_args; - auto_free char* exceptionFilter = NULL; auto_free char* fullDumpPath = NULL; auto_cancel_thread pthread_t waitForProfilerCompletion = -1; auto_free char* clientData = NULL; int rc = 0; - unsigned int clientDataSize = 0; - - enum TriggerType type = Exception; if ((rc = WaitForQuitOrEvent(config, &config->evtStartMonitoring, INFINITE_WAIT)) == WAIT_OBJECT_0 + 1) { - exceptionFilter = GetEncodedExceptionFilter(config->ExceptionFilter, config->NumberOfDumpsToCollect); - if(exceptionFilter==NULL) - { - Trace("ExceptionMonitoring: Failed to get exception filter."); - return NULL; - } - - if(config->CoreDumpName==NULL) + if(config->CoreDumpName == NULL) { // We don't have a dump name so we just use the path (append a '/' to indicate its a base path) if(config->CoreDumpPath[strlen(config->CoreDumpPath)-1] != '/') { fullDumpPath = malloc(strlen(config->CoreDumpPath) + 2); // +1 = '\0', +1 = '/' - if(fullDumpPath==NULL) + if(fullDumpPath == NULL) { - Trace("ExceptionMonitoring: Failed to allocate memory."); + Trace("DotNetMonitoringThread: Failed to allocate memory."); return NULL; } @@ -1275,9 +1283,9 @@ void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguratio else { fullDumpPath = malloc(strlen(config->CoreDumpPath) + 1); - if(fullDumpPath==NULL) + if(fullDumpPath == NULL) { - Trace("ExceptionMonitoring: Failed to allocate memory."); + Trace("DotNetMonitoringThread: Failed to allocate memory."); return NULL; } @@ -1290,9 +1298,9 @@ void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguratio if(config->CoreDumpPath[strlen(config->CoreDumpPath)] != '/') { fullDumpPath = malloc(strlen(config->CoreDumpPath) + strlen(config->CoreDumpName) + 2); // +1 = '\0', +1 = '/' - if(fullDumpPath==NULL) + if(fullDumpPath == NULL) { - Trace("ExceptionMonitoring: Failed to allocate memory."); + Trace("DotNetMonitoringThread: Failed to allocate memory."); return NULL; } @@ -1301,9 +1309,9 @@ void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguratio else { fullDumpPath = malloc(strlen(config->CoreDumpPath) + strlen(config->CoreDumpName) + 1); // +1 = '\0', +1 = '/' - if(fullDumpPath==NULL) + if(fullDumpPath == NULL) { - Trace("ExceptionMonitoring: Failed to allocate memory."); + Trace("DotNetMonitoringThread: Failed to allocate memory."); return NULL; } @@ -1314,7 +1322,7 @@ void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguratio // Create thread to wait for profiler completion if ((pthread_create(&waitForProfilerCompletion, NULL, WaitForProfilerCompletion, (void *) config)) != 0) { - Trace("ExceptionMonitoring: failed to create WaitForProfilerCompletion thread."); + Trace("DotNetMonitoringThread: failed to create WaitForProfilerCompletion thread."); return NULL; } @@ -1326,31 +1334,144 @@ void *ExceptionMonitoringThread(void *thread_args /* struct ProcDumpConfiguratio } pthread_mutex_unlock(&config->dotnetMutex); + // Get the corresponding client data to be sent to profiler + clientData = GetClientData(config, fullDumpPath); + if(clientData == NULL) + { + Trace("DotNetMonitoringThread: Failed to get client data."); + return NULL; + } + // Inject the profiler into the target process + if(InjectProfiler(config->ProcessId, clientData)!=0) + { + Trace("DotNetMonitoringThread: Failed to inject the profiler."); + pthread_cancel(waitForProfilerCompletion); + } + + pthread_join(waitForProfilerCompletion, NULL); + } - // client data in the following format: + Trace("DotNetMonitoringThread: Exiting DotNetMonitoringThread Thread"); + return NULL; +} + +//------------------------------------------------------------------------------------- +// +// GetClientData +// +// Gets the client data string depending on which triggers were requested in the +// specified config. +// +//------------------------------------------------------------------------------------- +char* GetClientData(struct ProcDumpConfiguration *self, char* fullDumpPath) +{ + Trace("GetClientData: Entering GetClientData"); + char* clientData = NULL; + auto_free char* exceptionFilter = NULL; + auto_free char* thresholds = NULL; + unsigned int clientDataSize = 0; + + if(self->bDumpOnException) + { // exception_trigger;;;:;:,... - clientDataSize = snprintf(NULL, 0, "%d;%s;%d;%s", type, fullDumpPath, getpid(), exceptionFilter) + 1; + exceptionFilter = GetEncodedExceptionFilter(self->ExceptionFilter, self->NumberOfDumpsToCollect); + if(exceptionFilter == NULL) + { + Trace("GetClientData: Failed to get exception filter."); + return NULL; + } + + clientDataSize = snprintf(NULL, 0, "%d;%s;%d;%s", Exception, fullDumpPath, getpid(), exceptionFilter) + 1; clientData = malloc(clientDataSize); - if(clientData==NULL) + if(clientData == NULL) { - Trace("ExceptionMonitoringThread: Failed to allocate memory for client data."); + Trace("GetClientData: Failed to allocate memory for client data."); return NULL; } - sprintf(clientData, "%d;%s;%d;%s", type, fullDumpPath, getpid(), exceptionFilter); + sprintf(clientData, "%d;%s;%d;%s", Exception, fullDumpPath, getpid(), exceptionFilter); + } + else if (self->bMonitoringGCMemory) + { + // GC Memory trigger (-gcm);;;Threshold1;Threshold2,... + thresholds = GetThresholds(self); + if(thresholds == NULL) + { + Trace("GetClientData: Failed to get thresholds."); + return NULL; + } - if(InjectProfiler(config->ProcessId, clientData)!=0) + clientDataSize = snprintf(NULL, 0, "%d;%s;%d;%s", GCThreshold, fullDumpPath, getpid(), thresholds) + 1; + clientData = malloc(clientDataSize); + if(clientData == NULL) { - Trace("ExceptionMonitoring: Failed to inject the profiler."); - pthread_cancel(waitForProfilerCompletion); + Trace("GetClientData: Failed to allocate memory for client data."); + return NULL; } - pthread_join(waitForProfilerCompletion, NULL); + sprintf(clientData, "%d;%s;%d;%s", GCThreshold, fullDumpPath, getpid(), thresholds); + } + else + { + Trace("GetClientData: Invalid trigger specified"); + return NULL; } - Trace("ExceptionMonitoring: Exiting ExceptionMonitoring Thread"); - return NULL; + Trace("GetClientData: Exiting GetClientData"); + return clientData; +} + +//------------------------------------------------------------------------------------- +// +// GetThresholds +// +// Returns a comma separated string of GC mem thresholds specified in +// self->MemoryThresholds +// +//------------------------------------------------------------------------------------- +char* GetThresholds(struct ProcDumpConfiguration *self) +{ + Trace("GetThresholds: Entering GetThresholds"); + int thresholdLen = 0; + char* thresholds = NULL; + + for(int i = 0; i < self->MemoryThresholdCount; i++) + { + thresholdLen += snprintf(NULL, 0, "%d", self->MemoryThreshold[i]); + if(i != self->MemoryThresholdCount - 1) + { + thresholdLen++; // Comma + } + } + + thresholdLen++; // NULL terminator + + thresholds = malloc(thresholdLen); + if(thresholds != NULL) + { + char* writePos = thresholds; + if(thresholds != NULL) + { + for(int i = 0; i < self->MemoryThresholdCount; i++) + { + int len = snprintf(writePos, thresholdLen, "%d", self->MemoryThreshold[i]); + writePos += len; + thresholdLen -= len; + if(i != self->MemoryThresholdCount - 1) + { + *writePos = ';'; + writePos++; + thresholdLen--; + } + } + } + + *(writePos) = '\0'; + } + + Trace("GetThresholds: Exiting GetThresholds"); + return thresholds; } //------------------------------------------------------------------------------------- diff --git a/src/ProcDumpConfiguration.c b/src/ProcDumpConfiguration.c index f5993d4..f104cd6 100644 --- a/src/ProcDumpConfiguration.c +++ b/src/ProcDumpConfiguration.c @@ -148,7 +148,9 @@ void InitProcDumpConfiguration(struct ProcDumpConfiguration *self) self->CpuThreshold = -1; self->bCpuTriggerBelowValue = false; self->MemoryThreshold = NULL; + self->MemoryThresholdCount = -1; self->MemoryCurrentThreshold = 0; + self->bMonitoringGCMemory = false; self->ThreadThreshold = -1; self->FileDescriptorThreshold = -1; self->SignalNumber = -1; @@ -295,6 +297,8 @@ struct ProcDumpConfiguration * CopyProcDumpConfiguration(struct ProcDumpConfigur } copy->bMemoryTriggerBelowValue = self->bMemoryTriggerBelowValue; + copy->MemoryThresholdCount = self->MemoryThresholdCount; + copy->bMonitoringGCMemory = self->bMonitoringGCMemory; copy->ThresholdSeconds = self->ThresholdSeconds; copy->bTimerThreshold = self->bTimerThreshold; copy->NumberOfDumpsToCollect = self->NumberOfDumpsToCollect; @@ -329,7 +333,7 @@ struct ProcDumpConfiguration * CopyProcDumpConfiguration(struct ProcDumpConfigur int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) { bool bProcessSpecified = false; - int numThresholds = -1; + int monitorDotnet = false; if (argc < 2) { Trace("GetOptions: Invalid number of command line arguments."); @@ -368,12 +372,12 @@ int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) 0 == strcasecmp( argv[i], "/ml" ) || 0 == strcasecmp( argv[i], "-ml" )) { - if( i+1 >= argc || numThresholds != -1 ) return PrintUsage(); - self->MemoryThreshold = GetSeparatedValues(argv[i+1], ",", &numThresholds); + if( i+1 >= argc || self->MemoryThresholdCount != -1 ) return PrintUsage(); + self->MemoryThreshold = GetSeparatedValues(argv[i+1], ",", &self->MemoryThresholdCount); - if(self->MemoryThreshold == NULL || numThresholds == 0) return PrintUsage(); + if(self->MemoryThreshold == NULL || self->MemoryThresholdCount == 0) return PrintUsage(); - for(int i = 0; i < numThresholds; i++) + for(int i = 0; i < self->MemoryThresholdCount; i++) { if(self->MemoryThreshold[i] < 0) { @@ -390,6 +394,27 @@ int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) i++; } + else if( 0 == strcasecmp( argv[i], "/gcm" ) || + 0 == strcasecmp( argv[i], "-gcm" )) + { + if( i+1 >= argc || self->MemoryThresholdCount != -1 || monitorDotnet == true ) return PrintUsage(); + self->MemoryThreshold = GetSeparatedValues(argv[i+1], ",", &self->MemoryThresholdCount); + + if(self->MemoryThreshold == NULL || self->MemoryThresholdCount == 0) return PrintUsage(); + + for(int i = 0; i < self->MemoryThresholdCount; i++) + { + if(self->MemoryThreshold[i] < 0) + { + Log(error, "Invalid memory threshold specified."); + free(self->MemoryThreshold); + return PrintUsage(); + } + } + + self->bMonitoringGCMemory = true; + i++; + } else if( 0 == strcasecmp( argv[i], "/tc" ) || 0 == strcasecmp( argv[i], "-tc" )) { @@ -482,6 +507,7 @@ int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) else if( 0 == strcasecmp( argv[i], "/e" ) || 0 == strcasecmp( argv[i], "-e" )) { + if( i+1 >= argc || monitorDotnet == true ) return PrintUsage(); self->bDumpOnException = true; } else if( 0 == strcasecmp( argv[i], "/f" ) || @@ -643,15 +669,15 @@ int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) // // Ensure consistency between number of thresholds specified and the -n switch - if(numThresholds > 1 && self->NumberOfDumpsToCollect != -1) + if(self->MemoryThresholdCount > 1 && self->NumberOfDumpsToCollect != -1) { Log(error, "When specifying more than one memory threshold the number of dumps switch (-n) is invalid."); return PrintUsage(); } - if(numThresholds != -1) + if(self->MemoryThresholdCount != -1) { - self->NumberOfDumpsToCollect = numThresholds; + self->NumberOfDumpsToCollect = self->MemoryThresholdCount; } // If exception filter is provided with no -e switch exit @@ -776,7 +802,14 @@ bool PrintConfiguration(struct ProcDumpConfiguration *self) } else { - printf("%-40s>= ", "Commit Threshold:"); + if(self->bMonitoringGCMemory == true) + { + printf("%-40s>= ", ".NET GC Commit Threshold:"); + } + else + { + printf("%-40s>= ", "Commit Threshold:"); + } } for(int i=0; iNumberOfDumpsToCollect; i++) @@ -887,6 +920,7 @@ int PrintUsage() printf(" [-s Seconds]\n"); printf(" [-c|-cl CPU_Usage]\n"); printf(" [-m|-ml Commit_Usage1[,Commit_Usage2...]]\n"); + printf(" [-gcm Commit_Usage1[,Commit_Usage2...]]\n"); printf(" [-tc Thread_Threshold]\n"); printf(" [-fc FileDescriptor_Threshold]\n"); printf(" [-sig Signal_Number]\n"); @@ -906,6 +940,7 @@ int PrintUsage() printf(" -cl CPU threshold below which to create a dump of the process.\n"); printf(" -m Memory commit threshold in MB at which to create a dump.\n"); printf(" -ml Trigger when memory commit drops below specified MB value.\n"); + printf(" -gcm [.NET] GC memory commit threshold in MB at which to create a dump.\n"); printf(" -tc Thread count threshold above which to create a dump of the process.\n"); printf(" -fc File descriptor count threshold above which to create a dump of the process.\n"); printf(" -sig Signal number to intercept to create a dump of the process.\n"); diff --git a/tests/integration/TestWebApi/Program.cs b/tests/integration/TestWebApi/Program.cs index 286baa1..4189d47 100644 --- a/tests/integration/TestWebApi/Program.cs +++ b/tests/integration/TestWebApi/Program.cs @@ -6,6 +6,12 @@ throw new System.InvalidOperationException(); }); +app.MapGet("/fullgc", () => +{ + System.GC.Collect(); +}); + + app.MapGet("/throwandcatchinvalidoperation", () => { try From 99a8167b003177f24e123aa885dd6d3cce3becbd Mon Sep 17 00:00:00 2001 From: Mario Hewardt Date: Thu, 6 Jul 2023 16:11:37 -0700 Subject: [PATCH 3/8] Feature complete --- profiler/inc/ProcDumpProfiler.h | 9 +- profiler/src/ProcDumpProfiler.cpp | 178 +++++++++++------- src/Monitor.c | 2 +- tests/integration/TestWebApi/Program.cs | 11 ++ tests/integration/helpers.sh | 0 .../scenarios/dotnet_1_gc_threshold_1_dump.sh | 73 +++++++ .../dotnet_3_gc_thresholds_3_dumps.sh | 73 +++++++ 7 files changed, 273 insertions(+), 73 deletions(-) mode change 100644 => 100755 tests/integration/helpers.sh create mode 100755 tests/integration/scenarios/dotnet_1_gc_threshold_1_dump.sh create mode 100755 tests/integration/scenarios/dotnet_3_gc_thresholds_3_dumps.sh diff --git a/profiler/inc/ProcDumpProfiler.h b/profiler/inc/ProcDumpProfiler.h index 171a5d1..0fee71d 100644 --- a/profiler/inc/ProcDumpProfiler.h +++ b/profiler/inc/ProcDumpProfiler.h @@ -104,18 +104,20 @@ class CorProfiler : public ICorProfilerCallback8 ICorProfilerInfo* corProfilerInfo; ICorProfilerInfo8* corProfilerInfo8; std::vector exceptionMonitorList; - std::vector gcMemoryThresholdMonitorList; + std::vector gcMemoryThresholdMonitorList; pthread_t healthThread; std::string processName; std::string fullDumpPath; pthread_mutex_t endDumpCondition; enum TriggerType triggerType; + int currentThresholdIndex; + bool gen2Collection; String GetExceptionName(ObjectID objectId); String GetExceptionMessage(ObjectID objectId); - bool ParseClientData(char* fw); + bool ParseClientData(char* clientData); WCHAR* GetUint16(char* buffer); - std::string GetDumpName(uint16_t dumpCount,std::string exceptionName); + std::string GetDumpName(uint16_t dumpCount,std::string name); std::string GetProcessName(); bool GenerateCoreClrDump(char* socketName, char* dumpFileName); bool IsCoreClrProcess(pid_t pid, char** socketName); @@ -126,6 +128,7 @@ class CorProfiler : public ICorProfilerCallback8 int recv_all(int socket, void* buffer, size_t length); bool WildcardSearch(WCHAR*, WCHAR*); u_int64_t GetGCHeapSize(); + bool WriteDumpHelper(std::string dumpName); public: CorProfiler(); diff --git a/profiler/src/ProcDumpProfiler.cpp b/profiler/src/ProcDumpProfiler.cpp index e116d0e..cb56171 100644 --- a/profiler/src/ProcDumpProfiler.cpp +++ b/profiler/src/ProcDumpProfiler.cpp @@ -192,7 +192,7 @@ bool CorProfiler::WildcardSearch(WCHAR* szClassName, WCHAR* szSearch) //------------------------------------------------------------------------------------------------------------------------------------------------------ // CorProfiler::CorProfiler //------------------------------------------------------------------------------------------------------------------------------------------------------ -CorProfiler::CorProfiler() : refCount(0), corProfilerInfo8(nullptr), corProfilerInfo3(nullptr), corProfilerInfo(nullptr), procDumpPid(0) +CorProfiler::CorProfiler() : refCount(0), corProfilerInfo8(nullptr), corProfilerInfo3(nullptr), corProfilerInfo(nullptr), procDumpPid(0), currentThresholdIndex(0) { // Configure logging el::Loggers::reconfigureAllLoggers(el::ConfigurationType::Filename, LOG_FILE); @@ -331,30 +331,30 @@ WCHAR* CorProfiler::GetUint16(char* buffer) // Syntax of client data: ;... // // DOTNET_EXCEPTION_TRIGGER;;;:;:,... +// DOTNET_GC_THRESHOLD_TRIGGER;;;Threshold1;Threshold2,... // //------------------------------------------------------------------------------------------------------------------------------------------------------ -bool CorProfiler::ParseClientData(char* filter) +bool CorProfiler::ParseClientData(char* clientData) { LOG(TRACE) << "CorProfiler::ParseClientData: Enter"; - std::stringstream exceptionFilter(filter); + std::stringstream clientDataStream(clientData); std::string segment; - std::vector exclist; + std::vector dataList; - while(std::getline(exceptionFilter, segment, ';')) + while(std::getline(clientDataStream, segment, ';')) { - exclist.push_back(segment); + dataList.push_back(segment); } - std::vector exceptionList; int i=0; - for(std::string exception : exclist) + for(std::string dataItem : dataList) { if(i == 0) { // // First part of list is the type of trigger that is being invoked. // - triggerType = static_cast(std::stoi(exception)); + triggerType = static_cast(std::stoi(dataItem)); LOG(TRACE) << "CorProfiler::ParseClientData: trigger type = " << triggerType; i++; continue; @@ -366,7 +366,7 @@ bool CorProfiler::ParseClientData(char* filter) // > base path to dump location (if it ends with '/') // > full path to dump file // - fullDumpPath = exception; + fullDumpPath = dataItem; LOG(TRACE) << "CorProfiler::ParseClientData: Full path to dump = " << fullDumpPath; i++; @@ -378,7 +378,7 @@ bool CorProfiler::ParseClientData(char* filter) // Third part of the list is always the procdump pid. // we we need this to communicate back status to procdump // - procDumpPid = std::stoi(exception); + procDumpPid = std::stoi(dataItem); LOG(TRACE) << "CorProfiler::ParseClientData: ProcDump PID = " << procDumpPid; i++; continue; @@ -388,7 +388,7 @@ bool CorProfiler::ParseClientData(char* filter) { // exception filter std::string segment2; - std::stringstream stream(exception); + std::stringstream stream(dataItem); ExceptionMonitorEntry entry; entry.exceptionID = NULL; entry.collectedDumps = 0; @@ -402,7 +402,8 @@ bool CorProfiler::ParseClientData(char* filter) } else if (triggerType == GCThreshold) { - gcMemoryThresholdMonitorList.push_back(std::stoi(exception)); + // GC threshold list + gcMemoryThresholdMonitorList.push_back(std::stoi(dataItem) << 20); } } @@ -413,7 +414,7 @@ bool CorProfiler::ParseClientData(char* filter) for (auto & element : gcMemoryThresholdMonitorList) { - LOG(TRACE) << "CorProfiler::ParseClientData:GCMemoryThreshold filter " << element; + LOG(TRACE) << "CorProfiler::ParseClientData:GCMemoryThreshold " << element; } LOG(TRACE) << "CorProfiler::ParseClientData: Exit"; @@ -643,14 +644,14 @@ std::string CorProfiler::GetProcessName() //------------------------------------------------------------------------------------------------------------------------------------------------------ // CorProfiler::GetDumpName //------------------------------------------------------------------------------------------------------------------------------------------------------ -std::string CorProfiler::GetDumpName(u_int16_t dumpCount,std::string exceptionName) +std::string CorProfiler::GetDumpName(u_int16_t dumpCount,std::string name) { LOG(TRACE) << "CorProfiler::GetDumpName: Enter"; std::ostringstream tmp; // // If the path ends in '/' it means we have a base path and need to create the full path according to: - // /_exception_ + // /___ // if(fullDumpPath[fullDumpPath.length()-1] == '/') { @@ -670,7 +671,7 @@ std::string CorProfiler::GetDumpName(u_int16_t dumpCount,std::string exceptionNa strftime(date, 26, "%Y-%m-%d_%H:%M:%S", timerInfo); LOG(TRACE) << "CorProfiler::GetDumpName: Date/time " << date; - tmp << fullDumpPath << processName.c_str() << "_" << dumpCount << "_" << exceptionName << "_" << date; + tmp << fullDumpPath << processName.c_str() << "_" << dumpCount << "_" << name << "_" << date; LOG(TRACE) << "CorProfiler::GetDumpName: Full path name " << tmp.str(); } else @@ -734,17 +735,57 @@ uint64_t CorProfiler::GetGCHeapSize() } } - LOG(TRACE) << "#######################CorProfiler::GetGCHeapSize: nObjectRanges " << nObjectRanges; - for (int i = nObjectRanges - 1; i >= 0; i--) { - LOG(TRACE) << "Range " << i << " Length " << pObjectRanges[i].rangeLength; gcHeapSize += pObjectRanges[i].rangeLength; } + LOG(TRACE) << "CorProfiler::GetGCHeapSize: Exit"; return gcHeapSize; } +//------------------------------------------------------------------------------------------------------------------------------------------------------ +// CorProfiler::WriteDumpHelper +//------------------------------------------------------------------------------------------------------------------------------------------------------ +bool CorProfiler::WriteDumpHelper(std::string dumpName) +{ + LOG(TRACE) << "CorProfiler::WriteDumpHelper: Enter"; + AutoMutex lock = AutoMutex(&endDumpCondition); + + char* socketName = NULL; + if(IsCoreClrProcess(getpid(), &socketName)) + { + LOG(TRACE) << "CorProfiler::WriteDumpHelper: Target is .NET process"; + bool res = GenerateCoreClrDump(socketName, const_cast (dumpName.c_str())); + if(res == false) + { + LOG(TRACE) << "CorProfiler::WriteDumpHelper: Failed to generate core dump"; + delete[] socketName; + return false; + } + + // Notify procdump that a dump was generated + if(SendDumpCompletedStatus(dumpName, PROFILER_STATUS_SUCCESS) == -1) + { + LOG(TRACE) << "CorProfiler::WriteDumpHelper: Failed to notify procdump about dump creation"; + delete[] socketName; + return false; + } + + LOG(TRACE) << "CorProfiler::WriteDumpHelper: Generating core dump result: " << res; + + delete[] socketName; + } + else + { + LOG(TRACE) << "CorProfiler::WriteDumpHelper: Target process is not a .NET process"; + return false; + } + + LOG(TRACE) << "CorProfiler::WriteDumpHelper: Exit"; + return true; +} + //------------------------------------------------------------------------------------------------------------------------------------------------------ // CorProfiler::GarbageCollectionStarted //------------------------------------------------------------------------------------------------------------------------------------------------------ @@ -752,9 +793,10 @@ HRESULT STDMETHODCALLTYPE CorProfiler::GarbageCollectionStarted(int cGenerations { LOG(TRACE) << "CorProfiler::GarbageCollectionStarted: Enter"; - uint64_t size = GetGCHeapSize(); - - LOG(TRACE) << "CorProfiler::GarbageCollectionStarted: Total heap size " << size; + if(generationCollected[2] == true) + { + gen2Collection = true; + } LOG(TRACE) << "CorProfiler::GarbageCollectionStarted: Exit"; return S_OK; @@ -767,9 +809,39 @@ HRESULT STDMETHODCALLTYPE CorProfiler::GarbageCollectionFinished() { LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Enter"; - uint64_t size = GetGCHeapSize(); + if(gen2Collection == true) + { + // We only want to check heap sizes and thresholds after a gen2 collection + gen2Collection = false; + uint64_t heapSize = GetGCHeapSize(); + LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Total heap size " << heapSize; + + if(currentThresholdIndex < gcMemoryThresholdMonitorList.size() && heapSize >= gcMemoryThresholdMonitorList[currentThresholdIndex]) + { + LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Current threshold value " << gcMemoryThresholdMonitorList[currentThresholdIndex]; + + std::string dump = GetDumpName(currentThresholdIndex + 1,convertString(L"gc_size")); + + // Generate dump + if(WriteDumpHelper(dump) == false) + { + SendCatastrophicFailureStatus(); + return E_FAIL; + } + else + { + currentThresholdIndex++; + } - LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Total heap size " << size; + if(currentThresholdIndex >= gcMemoryThresholdMonitorList.size()) + { + // Stop monitoring + LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Reached last threshold "; + CleanupProfiler(); + UnloadProfiler(); + } + } + } LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Exit"; return S_OK; @@ -811,7 +883,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionThrown(ObjectID thrownObjectId) for (auto & element : exceptionMonitorList) { WCHAR* exception = GetUint16(const_cast (element.exception.c_str())); - if(exception==NULL) + if(exception == NULL) { LOG(TRACE) << "CorProfiler::ExceptionThrown: Unable to get exception name (WHCAR)."; SendCatastrophicFailureStatus(); @@ -823,10 +895,10 @@ HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionThrown(ObjectID thrownObjectId) // // We have to serialize calls to the diag pipe to avoid concurrency issues // - AutoMutex lock = AutoMutex(&endDumpCondition); if(element.collectedDumps == element.dumpsToCollect) { LOG(TRACE) << "CorProfiler::ExceptionThrown: Dump count has been reached...exiting early"; + free(exception); return S_OK; } @@ -834,53 +906,21 @@ HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionThrown(ObjectID thrownObjectId) std::string dump = GetDumpName(element.collectedDumps,convertString(exceptionName.ToWString())); - // Invoke coreclr dump generation - char* socketName = NULL; - if(IsCoreClrProcess(getpid(), &socketName)) + if(WriteDumpHelper(dump) == false) { - LOG(TRACE) << "CorProfiler::ExceptionThrown: Target is .NET process"; - bool res = GenerateCoreClrDump(socketName, const_cast (dump.c_str())); - if(res==false) - { - delete[] socketName; - - // If we fail to generate a core dump we consider that a catastrophic failure and terminate all monitoring - SendCatastrophicFailureStatus(); - return E_FAIL; - } - - // - // In asp.net, an exception thrown in the app is caught by asp.net and then rethrown and caught again. To avoid - // generating multiple dumps for the same exception instance we store the objectID. If we have already generated - // a dump for that object ID we simply ignore it - // - element.exceptionID = thrownObjectId; - - // Notify procdump that a dump was generated - if(SendDumpCompletedStatus(dump, PROFILER_STATUS_SUCCESS)==-1) - { - delete[] socketName; - - SendCatastrophicFailureStatus(); - return E_FAIL; - } - - LOG(TRACE) << "CorProfiler::ExceptionThrown: Generating core dump result: " << res; - - if(res) - { - element.collectedDumps++; - } - - delete[] socketName; - } - else - { - LOG(TRACE) << "CorProfiler::ExceptionThrown: Unable to create dump"; SendCatastrophicFailureStatus(); + free(exception); return E_FAIL; } + // + // In asp.net, an exception thrown in the app is caught by asp.net and then rethrown and caught again. To avoid + // generating multiple dumps for the same exception instance we store the objectID. If we have already generated + // a dump for that object ID we simply ignore it + // + element.exceptionID = thrownObjectId; + + element.collectedDumps++; if(element.collectedDumps == element.dumpsToCollect) { // We're done collecting dumps... diff --git a/src/Monitor.c b/src/Monitor.c index faa2a34..e25cf45 100644 --- a/src/Monitor.c +++ b/src/Monitor.c @@ -566,7 +566,7 @@ int CreateMonitorThreads(struct ProcDumpConfiguration *self) } } - if (self->MemoryThreshold != NULL && !tooManyTriggers) + if (self->MemoryThreshold != NULL && !tooManyTriggers && self->bMonitoringGCMemory == false) { if (self->nThreads < MAX_TRIGGERS) { diff --git a/tests/integration/TestWebApi/Program.cs b/tests/integration/TestWebApi/Program.cs index 4189d47..6fafb5b 100644 --- a/tests/integration/TestWebApi/Program.cs +++ b/tests/integration/TestWebApi/Program.cs @@ -11,6 +11,17 @@ System.GC.Collect(); }); +app.MapGet("/memincrease", () => +{ + var myList = new List(); + myList.Add(new byte[15000000]); + System.GC.Collect(); + myList.Add(new byte[15000000]); + System.GC.Collect(); + myList.Add(new byte[15000000]); + System.GC.Collect(); +}); + app.MapGet("/throwandcatchinvalidoperation", () => { diff --git a/tests/integration/helpers.sh b/tests/integration/helpers.sh old mode 100644 new mode 100755 diff --git a/tests/integration/scenarios/dotnet_1_gc_threshold_1_dump.sh b/tests/integration/scenarios/dotnet_1_gc_threshold_1_dump.sh new file mode 100755 index 0000000..8fb3c47 --- /dev/null +++ b/tests/integration/scenarios/dotnet_1_gc_threshold_1_dump.sh @@ -0,0 +1,73 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; +PROCDUMPPATH=$(readlink -m "$DIR/../../../bin/procdump"); +TESTWEBAPIPATH=$(readlink -m "$DIR/../TestWebApi"); +HELPERS=$(readlink -m "$DIR/../helpers.sh"); + +source $HELPERS + +pushd . +cd $TESTWEBAPIPATH +rm -rf *TestWebApi_*gc_size_* +dotnet run --urls=http://localhost:5032& + +# waiting TestWebApi ready to service +waitforurl http://localhost:5032/throwinvalidoperation +if [ $? -eq -1 ]; then + pkill -9 TestWebApi + popds + exit 1 +fi + +# Wait for 3 GC commit thresholds (10, 20 and 30MB) +sudo $PROCDUMPPATH -log -gcm 10 -w TestWebApi& + +# waiting for procdump child process +PROCDUMPCHILDPID=-1 +waitforprocdump PROCDUMPCHILDPID +if [ $PROCDUMPCHILDPID -eq -1 ]; then + pkill -9 TestWebApi + pkill -9 procdump + popd + exit 1 +fi + +TESTCHILDPID=$(ps -o pid= -C "TestWebApi" | tr -d ' ') + +#make sure procdump ready to capture before throw exception by checking if socket created +SOCKETPATH=-1 +waitforprocdumpsocket $PROCDUMPCHILDPID $TESTCHILDPID SOCKETPATH +if [ $SOCKETPATH -eq -1 ]; then + pkill -9 TestWebApi + pkill -9 procdump + popd + exit 1 +fi +echo "SOCKETPATH: "$SOCKETPATH + +wget -O /dev/null http://localhost:5032/memincrease + +sudo pkill -9 procdump +COUNT=( $(ls *TestWebApi_*gc_size_* | wc -l) ) +if [ -S $SOCKETPATH ]; +then + rm $SOCKETPATH +fi + +if [[ "$COUNT" -eq 1 ]]; then + rm -rf *TestWebApi_*gc_size_* + popd + + #check to make sure profiler so is unloaded + PROF="$(cat /proc/${TESTCHILDPID}/maps | awk '{print $6}' | grep '\procdumpprofiler.so' | uniq)" + pkill -9 TestWebApi + if [[ "$PROF" == "procdumpprofiler.so" ]]; then + exit 1 + else + exit 0 + fi +else + pkill -9 TestWebApi + popd + exit 1 +fi diff --git a/tests/integration/scenarios/dotnet_3_gc_thresholds_3_dumps.sh b/tests/integration/scenarios/dotnet_3_gc_thresholds_3_dumps.sh new file mode 100755 index 0000000..4b721c4 --- /dev/null +++ b/tests/integration/scenarios/dotnet_3_gc_thresholds_3_dumps.sh @@ -0,0 +1,73 @@ +#!/bin/bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; +PROCDUMPPATH=$(readlink -m "$DIR/../../../bin/procdump"); +TESTWEBAPIPATH=$(readlink -m "$DIR/../TestWebApi"); +HELPERS=$(readlink -m "$DIR/../helpers.sh"); + +source $HELPERS + +pushd . +cd $TESTWEBAPIPATH +rm -rf *TestWebApi_*gc_size_* +dotnet run --urls=http://localhost:5032& + +# waiting TestWebApi ready to service +waitforurl http://localhost:5032/throwinvalidoperation +if [ $? -eq -1 ]; then + pkill -9 TestWebApi + popds + exit 1 +fi + +# Wait for 3 GC commit thresholds (10, 20 and 30MB) +sudo $PROCDUMPPATH -log -gcm 10,20,30 -w TestWebApi& + +# waiting for procdump child process +PROCDUMPCHILDPID=-1 +waitforprocdump PROCDUMPCHILDPID +if [ $PROCDUMPCHILDPID -eq -1 ]; then + pkill -9 TestWebApi + pkill -9 procdump + popd + exit 1 +fi + +TESTCHILDPID=$(ps -o pid= -C "TestWebApi" | tr -d ' ') + +#make sure procdump ready to capture before throw exception by checking if socket created +SOCKETPATH=-1 +waitforprocdumpsocket $PROCDUMPCHILDPID $TESTCHILDPID SOCKETPATH +if [ $SOCKETPATH -eq -1 ]; then + pkill -9 TestWebApi + pkill -9 procdump + popd + exit 1 +fi +echo "SOCKETPATH: "$SOCKETPATH + +wget -O /dev/null http://localhost:5032/memincrease + +sudo pkill -9 procdump +COUNT=( $(ls *TestWebApi_*gc_size_* | wc -l) ) +if [ -S $SOCKETPATH ]; +then + rm $SOCKETPATH +fi + +if [[ "$COUNT" -eq 3 ]]; then + rm -rf *TestWebApi_*gc_size_* + popd + + #check to make sure profiler so is unloaded + PROF="$(cat /proc/${TESTCHILDPID}/maps | awk '{print $6}' | grep '\procdumpprofiler.so' | uniq)" + pkill -9 TestWebApi + if [[ "$PROF" == "procdumpprofiler.so" ]]; then + exit 1 + else + exit 0 + fi +else + pkill -9 TestWebApi + popd + exit 1 +fi From ff0b52563621d57a1c9f09fcfa8a32dc1a57df9c Mon Sep 17 00:00:00 2001 From: Mario Hewardt Date: Thu, 6 Jul 2023 16:15:41 -0700 Subject: [PATCH 4/8] Update README with new switch and example --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 9cf05b0..ed25424 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ procdump [-n Count] [-s Seconds] [-c|-cl CPU_Usage] [-m|-ml Commit_Usage1[,Commit_Usage2,...]] + [-gcm Commit_Usage1[,Commit_Usage2...]] [-tc Thread_Threshold] [-fc FileDescriptor_Threshold] [-sig Signal_Number] @@ -44,6 +45,7 @@ Options: -cl CPU threshold below which to create a dump of the process. -m Memory commit thresholds (MB) above which to create dumps. -ml Memory commit thresholds (MB) below which to create dumps. + -gcm [.NET] GC memory commit threshold in MB at which to create a dump. -tc Thread count threshold above which to create a dump of the process. -fc File descriptor count threshold above which to create a dump of the process. -sig Signal number to intercept to create a dump of the process. @@ -90,6 +92,10 @@ The following will create a core dump when memory usage is >= 100 MB followed by ``` sudo procdump -m 100,200 1234 ``` +The following will create a core dump when .NET memory usage is >= 100 MB followed by another dump when memory usage is >= 200MB. +``` +sudo procdump -gcm 100,200 1234 +``` The following will create a core dump in the `/tmp` directory immediately. ``` sudo procdump 1234 /tmp From ea0b1f3bb786f37c44ddc2f19f73f7b894056db5 Mon Sep 17 00:00:00 2001 From: Mario Hewardt Date: Thu, 6 Jul 2023 16:21:07 -0700 Subject: [PATCH 5/8] Fix memory leak --- profiler/src/ProcDumpProfiler.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/profiler/src/ProcDumpProfiler.cpp b/profiler/src/ProcDumpProfiler.cpp index cb56171..528e22e 100644 --- a/profiler/src/ProcDumpProfiler.cpp +++ b/profiler/src/ProcDumpProfiler.cpp @@ -731,6 +731,7 @@ uint64_t CorProfiler::GetGCHeapSize() if (FAILED(hr) || nObjectRanges != nObjectRanges2) { LOG(TRACE) << "CorProfiler::GetGCHeapSize: Failed to call GetGenerationBounds with allocated memory"; + delete[] pObjectRanges; return E_FAIL; } } @@ -740,6 +741,11 @@ uint64_t CorProfiler::GetGCHeapSize() gcHeapSize += pObjectRanges[i].rangeLength; } + if(fHeapAlloc == true) + { + delete[] pObjectRanges; + } + LOG(TRACE) << "CorProfiler::GetGCHeapSize: Exit"; return gcHeapSize; } From 0e4bce7d26aba5999c8e495dece8b68c343e8dd5 Mon Sep 17 00:00:00 2001 From: Mario Hewardt Date: Fri, 7 Jul 2023 11:05:36 -0700 Subject: [PATCH 6/8] Fixes --- Makefile | 3 +-- README.md | 8 +++--- include/Includes.h | 1 - profiler/inc/ProcDumpProfiler.h | 2 +- profiler/src/ProcDumpProfiler.cpp | 8 +++--- src/Monitor.c | 4 +-- src/ProcDumpConfiguration.c | 25 +++++++++++++------ .../scenarios/dotnet_1_gc_threshold_1_dump.sh | 2 +- 8 files changed, 30 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index 654a8bd..8e48ec4 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,6 @@ TESTOUT=$(BINDIR)/ProcDumpTestApplication # Profiler PROFSRCDIR=profiler/src PROFINCDIR=profiler/inc -PROFCOMMONINCDIR=include PROFCXXFLAGS ?= -DELPP_NO_DEFAULT_LOG_FILE -DELPP_THREAD_SAFE -g -pthread -shared --no-undefined -Wno-invalid-noreturn -Wno-pragma-pack -Wno-writable-strings -Wno-format-security -fPIC -fms-extensions -DHOST_64BIT -DBIT64 -DPAL_STDCPP_COMPAT -DPLATFORM_UNIX -std=c++11 PROFCLANG=clang++ @@ -55,7 +54,7 @@ install: cp procdump.1 $(DESTDIR)$(MANDIR) $(OBJDIR)/ProcDumpProfiler.so: $(PROFSRCDIR)/ClassFactory.cpp $(PROFSRCDIR)/ProcDumpProfiler.cpp $(PROFSRCDIR)/dllmain.cpp $(PROFSRCDIR)/corprof_i.cpp $(PROFSRCDIR)/easylogging++.cc | $(OBJDIR) - $(PROFCLANG) -o $@ $(PROFCXXFLAGS) -I $(PROFINCDIR) -I $(PROFCOMMONINCDIR) $^ + $(PROFCLANG) -o $@ $(PROFCXXFLAGS) -I $(PROFINCDIR) -I $(INCDIR) $^ ld -r -b binary -o $(OBJDIR)/ProcDumpProfiler.o $(OBJDIR)/ProcDumpProfiler.so $(OBJDIR)/%.o: $(SRCDIR)/%.c | $(OBJDIR) diff --git a/README.md b/README.md index ed25424..a00a1d6 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ procdump [-n Count] [-s Seconds] [-c|-cl CPU_Usage] [-m|-ml Commit_Usage1[,Commit_Usage2,...]] - [-gcm Commit_Usage1[,Commit_Usage2...]] + [-gcm Memory_Usage1[,Memory_Usage2...]] [-tc Thread_Threshold] [-fc FileDescriptor_Threshold] [-sig Signal_Number] @@ -43,9 +43,9 @@ Options: -s Consecutive seconds before dump is written (default is 10). -c CPU threshold above which to create a dump of the process. -cl CPU threshold below which to create a dump of the process. - -m Memory commit thresholds (MB) above which to create dumps. - -ml Memory commit thresholds (MB) below which to create dumps. - -gcm [.NET] GC memory commit threshold in MB at which to create a dump. + -m Memory commit threshold(s) (MB) above which to create dumps. + -ml Memory commit threshold(s) (MB) below which to create dumps. + -gcm [.NET] GC memory threshold(s) (MB) above which to create dumps. -tc Thread count threshold above which to create a dump of the process. -fc File descriptor count threshold above which to create a dump of the process. -sig Signal number to intercept to create a dump of the process. diff --git a/include/Includes.h b/include/Includes.h index 289e1dc..b4e29b7 100644 --- a/include/Includes.h +++ b/include/Includes.h @@ -10,4 +10,3 @@ #include "Process.h" #include "DotnetHelpers.h" #include "ProfilerHelpers.h" - diff --git a/profiler/inc/ProcDumpProfiler.h b/profiler/inc/ProcDumpProfiler.h index 0fee71d..d1f8596 100644 --- a/profiler/inc/ProcDumpProfiler.h +++ b/profiler/inc/ProcDumpProfiler.h @@ -117,7 +117,7 @@ class CorProfiler : public ICorProfilerCallback8 String GetExceptionMessage(ObjectID objectId); bool ParseClientData(char* clientData); WCHAR* GetUint16(char* buffer); - std::string GetDumpName(uint16_t dumpCount,std::string name); + std::string GetDumpName(uint16_t dumpCount, std::string name); std::string GetProcessName(); bool GenerateCoreClrDump(char* socketName, char* dumpFileName); bool IsCoreClrProcess(pid_t pid, char** socketName); diff --git a/profiler/src/ProcDumpProfiler.cpp b/profiler/src/ProcDumpProfiler.cpp index 528e22e..b890189 100644 --- a/profiler/src/ProcDumpProfiler.cpp +++ b/profiler/src/ProcDumpProfiler.cpp @@ -192,7 +192,7 @@ bool CorProfiler::WildcardSearch(WCHAR* szClassName, WCHAR* szSearch) //------------------------------------------------------------------------------------------------------------------------------------------------------ // CorProfiler::CorProfiler //------------------------------------------------------------------------------------------------------------------------------------------------------ -CorProfiler::CorProfiler() : refCount(0), corProfilerInfo8(nullptr), corProfilerInfo3(nullptr), corProfilerInfo(nullptr), procDumpPid(0), currentThresholdIndex(0) +CorProfiler::CorProfiler() : refCount(0), corProfilerInfo8(nullptr), corProfilerInfo3(nullptr), corProfilerInfo(nullptr), procDumpPid(0), currentThresholdIndex(0), gen2Collection(false) { // Configure logging el::Loggers::reconfigureAllLoggers(el::ConfigurationType::Filename, LOG_FILE); @@ -644,7 +644,7 @@ std::string CorProfiler::GetProcessName() //------------------------------------------------------------------------------------------------------------------------------------------------------ // CorProfiler::GetDumpName //------------------------------------------------------------------------------------------------------------------------------------------------------ -std::string CorProfiler::GetDumpName(u_int16_t dumpCount,std::string name) +std::string CorProfiler::GetDumpName(u_int16_t dumpCount, std::string name) { LOG(TRACE) << "CorProfiler::GetDumpName: Enter"; std::ostringstream tmp; @@ -799,7 +799,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::GarbageCollectionStarted(int cGenerations { LOG(TRACE) << "CorProfiler::GarbageCollectionStarted: Enter"; - if(generationCollected[2] == true) + if(gcMemoryThresholdMonitorList.size() > 0 && generationCollected[2] == true) { gen2Collection = true; } @@ -817,7 +817,7 @@ HRESULT STDMETHODCALLTYPE CorProfiler::GarbageCollectionFinished() if(gen2Collection == true) { - // We only want to check heap sizes and thresholds after a gen2 collection + // During a GC threshold trigger, we only want to check heap sizes and thresholds after a gen2 collection gen2Collection = false; uint64_t heapSize = GetGCHeapSize(); LOG(TRACE) << "CorProfiler::GarbageCollectionFinished: Total heap size " << heapSize; diff --git a/src/Monitor.c b/src/Monitor.c index e25cf45..bea135f 100644 --- a/src/Monitor.c +++ b/src/Monitor.c @@ -1426,8 +1426,8 @@ char* GetClientData(struct ProcDumpConfiguration *self, char* fullDumpPath) // // GetThresholds // -// Returns a comma separated string of GC mem thresholds specified in -// self->MemoryThresholds +// Returns a ; separated string of GC mem thresholds specified in +// self->MemoryThreshold // //------------------------------------------------------------------------------------- char* GetThresholds(struct ProcDumpConfiguration *self) diff --git a/src/ProcDumpConfiguration.c b/src/ProcDumpConfiguration.c index f104cd6..252e6b8 100644 --- a/src/ProcDumpConfiguration.c +++ b/src/ProcDumpConfiguration.c @@ -333,7 +333,7 @@ struct ProcDumpConfiguration * CopyProcDumpConfiguration(struct ProcDumpConfigur int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) { bool bProcessSpecified = false; - int monitorDotnet = false; + int dotnetTriggerCount = 0; if (argc < 2) { Trace("GetOptions: Invalid number of command line arguments."); @@ -397,7 +397,7 @@ int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) else if( 0 == strcasecmp( argv[i], "/gcm" ) || 0 == strcasecmp( argv[i], "-gcm" )) { - if( i+1 >= argc || self->MemoryThresholdCount != -1 || monitorDotnet == true ) return PrintUsage(); + if( i+1 >= argc || self->MemoryThresholdCount != -1) return PrintUsage(); self->MemoryThreshold = GetSeparatedValues(argv[i+1], ",", &self->MemoryThresholdCount); if(self->MemoryThreshold == NULL || self->MemoryThresholdCount == 0) return PrintUsage(); @@ -412,6 +412,7 @@ int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) } } + dotnetTriggerCount++; self->bMonitoringGCMemory = true; i++; } @@ -507,7 +508,8 @@ int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) else if( 0 == strcasecmp( argv[i], "/e" ) || 0 == strcasecmp( argv[i], "-e" )) { - if( i+1 >= argc || monitorDotnet == true ) return PrintUsage(); + if( i+1 >= argc) return PrintUsage(); + dotnetTriggerCount++; self->bDumpOnException = true; } else if( 0 == strcasecmp( argv[i], "/f" ) || @@ -668,6 +670,13 @@ int GetOptions(struct ProcDumpConfiguration *self, int argc, char *argv[]) // Validate multi arguments // + // .NET triggers are mutually exclusive + if(dotnetTriggerCount > 1) + { + Log(error, "Only one .NET trigger can be specified."); + return PrintUsage(); + } + // Ensure consistency between number of thresholds specified and the -n switch if(self->MemoryThresholdCount > 1 && self->NumberOfDumpsToCollect != -1) { @@ -804,7 +813,7 @@ bool PrintConfiguration(struct ProcDumpConfiguration *self) { if(self->bMonitoringGCMemory == true) { - printf("%-40s>= ", ".NET GC Commit Threshold:"); + printf("%-40s>= ", ".NET Memory Threshold:"); } else { @@ -920,7 +929,7 @@ int PrintUsage() printf(" [-s Seconds]\n"); printf(" [-c|-cl CPU_Usage]\n"); printf(" [-m|-ml Commit_Usage1[,Commit_Usage2...]]\n"); - printf(" [-gcm Commit_Usage1[,Commit_Usage2...]]\n"); + printf(" [-gcm Memory_Usage1[,Memory_Usage2...]]\n"); printf(" [-tc Thread_Threshold]\n"); printf(" [-fc FileDescriptor_Threshold]\n"); printf(" [-sig Signal_Number]\n"); @@ -938,9 +947,9 @@ int PrintUsage() printf(" -s Consecutive seconds before dump is written (default is 10).\n"); printf(" -c CPU threshold above which to create a dump of the process.\n"); printf(" -cl CPU threshold below which to create a dump of the process.\n"); - printf(" -m Memory commit threshold in MB at which to create a dump.\n"); - printf(" -ml Trigger when memory commit drops below specified MB value.\n"); - printf(" -gcm [.NET] GC memory commit threshold in MB at which to create a dump.\n"); + printf(" -m Memory commit threshold(s) (MB) above which to create dumps.\n"); + printf(" -ml Memory commit threshold(s) (MB) below which to create dumps.\n"); + printf(" -gcm [.NET] GC memory threshold(s) (MB) above which to create dumps.\n"); printf(" -tc Thread count threshold above which to create a dump of the process.\n"); printf(" -fc File descriptor count threshold above which to create a dump of the process.\n"); printf(" -sig Signal number to intercept to create a dump of the process.\n"); diff --git a/tests/integration/scenarios/dotnet_1_gc_threshold_1_dump.sh b/tests/integration/scenarios/dotnet_1_gc_threshold_1_dump.sh index 8fb3c47..f166eb6 100755 --- a/tests/integration/scenarios/dotnet_1_gc_threshold_1_dump.sh +++ b/tests/integration/scenarios/dotnet_1_gc_threshold_1_dump.sh @@ -19,7 +19,7 @@ if [ $? -eq -1 ]; then exit 1 fi -# Wait for 3 GC commit thresholds (10, 20 and 30MB) +# Wait for 1 GC commit thresholds (10MB) sudo $PROCDUMPPATH -log -gcm 10 -w TestWebApi& # waiting for procdump child process From 4aa225ccfee884b8c40d9cdf3ecc3df7437ee9a2 Mon Sep 17 00:00:00 2001 From: Mario Hewardt Date: Fri, 7 Jul 2023 16:40:16 -0700 Subject: [PATCH 7/8] PR feedback --- profiler/src/ProcDumpProfiler.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/profiler/src/ProcDumpProfiler.cpp b/profiler/src/ProcDumpProfiler.cpp index b890189..93eba20 100644 --- a/profiler/src/ProcDumpProfiler.cpp +++ b/profiler/src/ProcDumpProfiler.cpp @@ -357,9 +357,8 @@ bool CorProfiler::ParseClientData(char* clientData) triggerType = static_cast(std::stoi(dataItem)); LOG(TRACE) << "CorProfiler::ParseClientData: trigger type = " << triggerType; i++; - continue; } - if(i == 1) + else if(i == 1) { // // Second part of the list is either: @@ -370,9 +369,8 @@ bool CorProfiler::ParseClientData(char* clientData) LOG(TRACE) << "CorProfiler::ParseClientData: Full path to dump = " << fullDumpPath; i++; - continue; } - if(i == 2) + else if(i == 2) { // // Third part of the list is always the procdump pid. @@ -381,10 +379,8 @@ bool CorProfiler::ParseClientData(char* clientData) procDumpPid = std::stoi(dataItem); LOG(TRACE) << "CorProfiler::ParseClientData: ProcDump PID = " << procDumpPid; i++; - continue; } - - if(triggerType == Exception) + else if(triggerType == Exception) { // exception filter std::string segment2; @@ -405,6 +401,11 @@ bool CorProfiler::ParseClientData(char* clientData) // GC threshold list gcMemoryThresholdMonitorList.push_back(std::stoi(dataItem) << 20); } + else + { + LOG(TRACE) << "CorProfiler::ParseClientData Unrecognized trigger type"; + return false; + } } for (auto & element : exceptionMonitorList) From 1b5ee57013d3ade83c19c74189653c5f2b167c42 Mon Sep 17 00:00:00 2001 From: Mario Hewardt Date: Mon, 10 Jul 2023 10:48:42 -0700 Subject: [PATCH 8/8] PR feedback --- profiler/src/ProcDumpProfiler.cpp | 9 +++------ src/Monitor.c | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/profiler/src/ProcDumpProfiler.cpp b/profiler/src/ProcDumpProfiler.cpp index 93eba20..51b09e4 100644 --- a/profiler/src/ProcDumpProfiler.cpp +++ b/profiler/src/ProcDumpProfiler.cpp @@ -708,7 +708,7 @@ uint64_t CorProfiler::GetGCHeapSize() if (FAILED(hr)) { LOG(TRACE) << "CorProfiler::GetGCHeapSize: Failed calling GetGenerationBounds " << hr; - return E_FAIL; + return 0; } if (nObjectRanges <= cRanges) @@ -722,7 +722,7 @@ uint64_t CorProfiler::GetGCHeapSize() if (pObjectRanges == NULL) { LOG(TRACE) << "CorProfiler::GetGCHeapSize: Failed to allocate memory for object ranges "; - return E_FAIL; + return 0; } fHeapAlloc = true; @@ -733,7 +733,7 @@ uint64_t CorProfiler::GetGCHeapSize() { LOG(TRACE) << "CorProfiler::GetGCHeapSize: Failed to call GetGenerationBounds with allocated memory"; delete[] pObjectRanges; - return E_FAIL; + return 0; } } @@ -899,9 +899,6 @@ HRESULT STDMETHODCALLTYPE CorProfiler::ExceptionThrown(ObjectID thrownObjectId) if(WildcardSearch(exceptionWCHARs,exception) && element.exceptionID != thrownObjectId) { - // - // We have to serialize calls to the diag pipe to avoid concurrency issues - // if(element.collectedDumps == element.dumpsToCollect) { LOG(TRACE) << "CorProfiler::ExceptionThrown: Dump count has been reached...exiting early"; diff --git a/src/Monitor.c b/src/Monitor.c index bea135f..5adabdf 100644 --- a/src/Monitor.c +++ b/src/Monitor.c @@ -1250,7 +1250,7 @@ void *TimerThread(void *thread_args /* struct ProcDumpConfiguration* */) // // NOTE: .NET only. // NOTE: At the moment, .NET triggers are mutually exclusive meaning -// only one can specified at a time. For example, you cannot specify +// only one can be specified at a time. For example, you cannot specify // both exception based monitoring and GC based monitoring. // //--------------------------------------------------------------------