Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RFC: server: add mechanism for graceful shutdown. #3307

Closed
wants to merge 3 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions include/envoy/server/listener_manager.h
Original file line number Diff line number Diff line change
@@ -120,6 +120,8 @@ class ListenerManager {
*/
virtual void stopListeners() PURE;

virtual void closeListeners() PURE;

/**
* Stop all threaded workers from running. When this routine returns all worker threads will
* have exited.
8 changes: 7 additions & 1 deletion source/server/hot_restart_impl.cc
Original file line number Diff line number Diff line change
@@ -469,7 +469,13 @@ void HotRestartImpl::terminateParent() {
parent_terminated_ = true;
}

void HotRestartImpl::shutdown() { socket_event_.reset(); }
void HotRestartImpl::shutdown() {
socket_event_.reset();
if (my_domain_socket_ >= 0) {
::close(my_domain_socket_);
my_domain_socket_ = -1;
}
}

std::string HotRestartImpl::version() {
return versionHelper(shmem_.maxStats(), Stats::RawStatData::maxNameLength(), *stats_set_);
24 changes: 24 additions & 0 deletions source/server/http/admin.cc
Original file line number Diff line number Diff line change
@@ -318,6 +318,28 @@ Http::Code AdminImpl::handlerCpuProfiler(absl::string_view url, Http::HeaderMap&
return Http::Code::OK;
}

Http::Code AdminImpl::handlerGracefulShutdown(absl::string_view, Http::HeaderMap&,
Buffer::Instance& response) {
if (graceful_shutdown_timer_ != nullptr) {
response.add("Graceful shutdown already in progress.\n");
return Http::Code::BadRequest;
}

const std::chrono::seconds shutdown_time(server_.options().parentShutdownTime());
ENVOY_LOG(info, "Starting graceful shutdown. Server will exit in {} seconds.",
shutdown_time.count());
graceful_shutdown_timer_ = server_.dispatcher().createTimer([this]() { server_.shutdown(); });
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like if AdminImpl is destructed before the timeout is reached, the call to server_.shutdown() will crash. Can we cancel the timer in the destructor if non-null, and null the timer here?

Or is this not necessary due to the way the server_.dispatcher() works?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The timer is held in a unique_ptr. If it is destructed, the timer is cancelled/unscheduled.

graceful_shutdown_timer_->enableTimer(shutdown_time);

server_.drainListeners();
server_.listenerManager().closeListeners();
server_.shutdownAdmin();
server_.hotRestart().shutdown();

response.add("OK\n");
return Http::Code::OK;
}

Http::Code AdminImpl::handlerHealthcheckFail(absl::string_view, Http::HeaderMap&,
Buffer::Instance& response) {
server_.failHealthcheck(true);
@@ -751,6 +773,8 @@ AdminImpl::AdminImpl(const std::string& access_log_path, const std::string& prof
false, false},
{"/cpuprofiler", "enable/disable the CPU profiler",
MAKE_ADMIN_HANDLER(handlerCpuProfiler), false, true},
{"/graceful_shutdown", "close all listeners and drain connections",
MAKE_ADMIN_HANDLER(handlerGracefulShutdown), false, true},
{"/healthcheck/fail", "cause the server to fail health checks",
MAKE_ADMIN_HANDLER(handlerHealthcheckFail), false, true},
{"/healthcheck/ok", "cause the server to pass health checks",
3 changes: 3 additions & 0 deletions source/server/http/admin.h
Original file line number Diff line number Diff line change
@@ -152,6 +152,8 @@ class AdminImpl : public Admin,
Buffer::Instance& response) const;
Http::Code handlerCpuProfiler(absl::string_view path_and_query, Http::HeaderMap& response_headers,
Buffer::Instance& response);
Http::Code handlerGracefulShutdown(absl::string_view path_and_query,
Http::HeaderMap& response_headers, Buffer::Instance& response);
Http::Code handlerHealthcheckFail(absl::string_view path_and_query,
Http::HeaderMap& response_headers, Buffer::Instance& response);
Http::Code handlerHealthcheckOk(absl::string_view path_and_query,
@@ -225,6 +227,7 @@ class AdminImpl : public Admin,
AdminListener listener_;
Http::Http1Settings http1_settings_;
ConfigTrackerImpl config_tracker_;
Event::TimerPtr graceful_shutdown_timer_;
};

/**
14 changes: 14 additions & 0 deletions source/server/listener_manager_impl.cc
Original file line number Diff line number Diff line change
@@ -622,6 +622,20 @@ void ListenerManagerImpl::stopListeners() {
}
}

void ListenerManagerImpl::closeListeners() {
for (auto& listener : active_listeners_) {
listener->closeSocket();
}

for (auto& listener : warming_listeners_) {
listener->closeSocket();
}

for (auto& draining_listener : draining_listeners_) {
draining_listener.listener_->closeSocket();
}
}

void ListenerManagerImpl::stopWorkers() {
if (!workers_started_) {
return;
6 changes: 6 additions & 0 deletions source/server/listener_manager_impl.h
Original file line number Diff line number Diff line change
@@ -102,6 +102,7 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable<Logger::Id:
bool removeListener(const std::string& listener_name) override;
void startWorkers(GuardDog& guard_dog) override;
void stopListeners() override;
void closeListeners() override;
void stopWorkers() override;

Instance& server_;
@@ -203,6 +204,11 @@ class ListenerImpl : public Network::ListenerConfig,
return ret;
}

/**
* Close the listening socket.
*/
void closeSocket() { socket_->close(); }

Network::Address::InstanceConstSharedPtr address() const { return address_; }
const Network::SocketSharedPtr& getSocket() const { return socket_; }
void debugLog(const std::string& message);