/* * Copyright 2016, The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "debuggerd/handler.h" #include "dump_type.h" #include "protocol.h" #include "util.h" #include "intercept_manager.h" using android::base::GetIntProperty; using android::base::SendFileDescriptors; using android::base::StringPrintf; using android::base::borrowed_fd; using android::base::unique_fd; static InterceptManager* intercept_manager; enum CrashStatus { kCrashStatusRunning, kCrashStatusQueued, }; struct CrashArtifact { unique_fd fd; std::optional temporary_path; static CrashArtifact devnull() { CrashArtifact result; result.fd.reset(open("/dev/null", O_WRONLY | O_CLOEXEC)); return result; } }; struct CrashArtifactPaths { std::string text; std::optional proto; }; struct CrashOutput { CrashArtifact text; std::optional proto; }; // Ownership of Crash is a bit messy. // It's either owned by an active event that must have a timeout, or owned by // queued_requests, in the case that multiple crashes come in at the same time. struct Crash { ~Crash() { event_free(crash_event); } CrashOutput output; unique_fd crash_socket_fd; pid_t crash_pid; event* crash_event = nullptr; DebuggerdDumpType crash_type; }; class CrashQueue { public: CrashQueue(const std::string& dir_path, const std::string& file_name_prefix, size_t max_artifacts, size_t max_concurrent_dumps, bool supports_proto) : file_name_prefix_(file_name_prefix), dir_path_(dir_path), dir_fd_(open(dir_path.c_str(), O_DIRECTORY | O_RDONLY | O_CLOEXEC)), max_artifacts_(max_artifacts), next_artifact_(0), max_concurrent_dumps_(max_concurrent_dumps), num_concurrent_dumps_(0), supports_proto_(supports_proto) { if (dir_fd_ == -1) { PLOG(FATAL) << "failed to open directory: " << dir_path; } // NOTE: If max_artifacts_ <= max_concurrent_dumps_, then theoretically the // same filename could be handed out to multiple processes. CHECK(max_artifacts_ > max_concurrent_dumps_); find_oldest_artifact(); } static CrashQueue* for_crash(const Crash* crash) { return (crash->crash_type == kDebuggerdJavaBacktrace) ? for_anrs() : for_tombstones(); } static CrashQueue* for_crash(const std::unique_ptr& crash) { return for_crash(crash.get()); } static CrashQueue* for_tombstones() { static CrashQueue queue("/data/tombstones", "tombstone_" /* file_name_prefix */, GetIntProperty("tombstoned.max_tombstone_count", 32), 1 /* max_concurrent_dumps */, true /* supports_proto */); return &queue; } static CrashQueue* for_anrs() { static CrashQueue queue("/data/anr", "trace_" /* file_name_prefix */, GetIntProperty("tombstoned.max_anr_count", 64), 4 /* max_concurrent_dumps */, false /* supports_proto */); return &queue; } CrashArtifact create_temporary_file() const { CrashArtifact result; std::optional path; result.fd.reset(openat(dir_fd_, ".", O_WRONLY | O_APPEND | O_TMPFILE | O_CLOEXEC, 0660)); if (result.fd == -1) { // We might not have O_TMPFILE. Try creating with an arbitrary filename instead. static size_t counter = 0; std::string tmp_filename = StringPrintf(".temporary%zu", counter++); result.fd.reset(openat(dir_fd_, tmp_filename.c_str(), O_WRONLY | O_APPEND | O_CREAT | O_TRUNC | O_CLOEXEC, 0660)); if (result.fd == -1) { PLOG(FATAL) << "failed to create temporary tombstone in " << dir_path_; } result.temporary_path = std::move(tmp_filename); } return std::move(result); } std::optional get_output(DebuggerdDumpType dump_type) { CrashOutput result; switch (dump_type) { case kDebuggerdNativeBacktrace: // Don't generate tombstones for native backtrace requests. return {}; case kDebuggerdTombstoneProto: if (!supports_proto_) { LOG(ERROR) << "received kDebuggerdTombstoneProto on a queue that doesn't support proto"; return {}; } result.proto = create_temporary_file(); result.text = create_temporary_file(); break; case kDebuggerdJavaBacktrace: case kDebuggerdTombstone: result.text = create_temporary_file(); break; default: LOG(ERROR) << "unexpected dump type: " << dump_type; return {}; } return result; } borrowed_fd dir_fd() { return dir_fd_; } CrashArtifactPaths get_next_artifact_paths() { CrashArtifactPaths result; result.text = StringPrintf("%s%02d", file_name_prefix_.c_str(), next_artifact_); if (supports_proto_) { result.proto = StringPrintf("%s%02d.pb", file_name_prefix_.c_str(), next_artifact_); } next_artifact_ = (next_artifact_ + 1) % max_artifacts_; return result; } // Consumes crash if it returns true, otherwise leaves it untouched. bool maybe_enqueue_crash(std::unique_ptr&& crash) { if (num_concurrent_dumps_ == max_concurrent_dumps_) { queued_requests_.emplace_back(std::move(crash)); return true; } return false; } void maybe_dequeue_crashes(void (*handler)(std::unique_ptr crash)) { while (!queued_requests_.empty() && num_concurrent_dumps_ < max_concurrent_dumps_) { std::unique_ptr next_crash = std::move(queued_requests_.front()); queued_requests_.pop_front(); handler(std::move(next_crash)); } } void on_crash_started() { ++num_concurrent_dumps_; } void on_crash_completed() { --num_concurrent_dumps_; } private: void find_oldest_artifact() { size_t oldest_tombstone = 0; time_t oldest_time = std::numeric_limits::max(); for (size_t i = 0; i < max_artifacts_; ++i) { std::string path = StringPrintf("%s/%s%02zu", dir_path_.c_str(), file_name_prefix_.c_str(), i); struct stat st; if (stat(path.c_str(), &st) != 0) { if (errno == ENOENT) { oldest_tombstone = i; break; } else { PLOG(ERROR) << "failed to stat " << path; continue; } } if (st.st_mtime < oldest_time) { oldest_tombstone = i; oldest_time = st.st_mtime; } } next_artifact_ = oldest_tombstone; } const std::string file_name_prefix_; const std::string dir_path_; const int dir_fd_; const size_t max_artifacts_; int next_artifact_; const size_t max_concurrent_dumps_; size_t num_concurrent_dumps_; bool supports_proto_; std::deque> queued_requests_; DISALLOW_COPY_AND_ASSIGN(CrashQueue); }; // Whether java trace dumps are produced via tombstoned. static constexpr bool kJavaTraceDumpsEnabled = true; // Forward declare the callbacks so they can be placed in a sensible order. static void crash_accept_cb(evconnlistener* listener, evutil_socket_t sockfd, sockaddr*, int, void*); static void crash_request_cb(evutil_socket_t sockfd, short ev, void* arg); static void crash_completed_cb(evutil_socket_t sockfd, short ev, void* arg); static void perform_request(std::unique_ptr crash) { unique_fd output_fd; bool intercepted = intercept_manager->GetIntercept(crash->crash_pid, crash->crash_type, &output_fd); if (intercepted) { if (crash->crash_type == kDebuggerdTombstoneProto) { crash->output.proto = CrashArtifact::devnull(); } } else { if (auto o = CrashQueue::for_crash(crash.get())->get_output(crash->crash_type); o) { crash->output = std::move(*o); output_fd.reset(dup(crash->output.text.fd)); } else { LOG(ERROR) << "failed to get crash output for type " << crash->crash_type; return; } } TombstonedCrashPacket response = {.packet_type = CrashPacketType::kPerformDump}; ssize_t rc = -1; if (crash->output.proto) { rc = SendFileDescriptors(crash->crash_socket_fd, &response, sizeof(response), output_fd.get(), crash->output.proto->fd.get()); } else { rc = SendFileDescriptors(crash->crash_socket_fd, &response, sizeof(response), output_fd.get()); } output_fd.reset(); if (rc == -1) { PLOG(WARNING) << "failed to send response to CrashRequest"; return; } else if (rc != sizeof(response)) { PLOG(WARNING) << "crash socket write returned short"; return; } // TODO: Make this configurable by the interceptor? struct timeval timeout = {10 * android::base::HwTimeoutMultiplier(), 0}; event_base* base = event_get_base(crash->crash_event); event_assign(crash->crash_event, base, crash->crash_socket_fd, EV_TIMEOUT | EV_READ, crash_completed_cb, crash.get()); event_add(crash->crash_event, &timeout); CrashQueue::for_crash(crash)->on_crash_started(); // The crash is now owned by the event loop. crash.release(); } static void crash_accept_cb(evconnlistener* listener, evutil_socket_t sockfd, sockaddr*, int, void*) { event_base* base = evconnlistener_get_base(listener); Crash* crash = new Crash(); // TODO: Make sure that only java crashes come in on the java socket // and only native crashes on the native socket. struct timeval timeout = {1 * android::base::HwTimeoutMultiplier(), 0}; event* crash_event = event_new(base, sockfd, EV_TIMEOUT | EV_READ, crash_request_cb, crash); crash->crash_socket_fd.reset(sockfd); crash->crash_event = crash_event; event_add(crash_event, &timeout); } static void crash_request_cb(evutil_socket_t sockfd, short ev, void* arg) { std::unique_ptr crash(static_cast(arg)); TombstonedCrashPacket request = {}; if ((ev & EV_TIMEOUT) != 0) { LOG(WARNING) << "crash request timed out"; return; } else if ((ev & EV_READ) == 0) { LOG(WARNING) << "tombstoned received unexpected event from crash socket"; return; } ssize_t rc = TEMP_FAILURE_RETRY(read(sockfd, &request, sizeof(request))); if (rc == -1) { PLOG(WARNING) << "failed to read from crash socket"; return; } else if (rc != sizeof(request)) { LOG(WARNING) << "crash socket received short read of length " << rc << " (expected " << sizeof(request) << ")"; return; } if (request.packet_type != CrashPacketType::kDumpRequest) { LOG(WARNING) << "unexpected crash packet type, expected kDumpRequest, received " << StringPrintf("%#2hhX", request.packet_type); return; } crash->crash_type = request.packet.dump_request.dump_type; if (crash->crash_type < 0 || crash->crash_type > kDebuggerdTombstoneProto) { LOG(WARNING) << "unexpected crash dump type: " << crash->crash_type; return; } if (crash->crash_type != kDebuggerdJavaBacktrace) { crash->crash_pid = request.packet.dump_request.pid; } else { // Requests for java traces are sent from untrusted processes, so we // must not trust the PID sent down with the request. Instead, we ask the // kernel. ucred cr = {}; socklen_t len = sizeof(cr); int ret = getsockopt(sockfd, SOL_SOCKET, SO_PEERCRED, &cr, &len); if (ret != 0) { PLOG(ERROR) << "Failed to getsockopt(..SO_PEERCRED)"; return; } crash->crash_pid = cr.pid; } pid_t crash_pid = crash->crash_pid; LOG(INFO) << "received crash request for pid " << crash_pid; if (CrashQueue::for_crash(crash)->maybe_enqueue_crash(std::move(crash))) { LOG(INFO) << "enqueueing crash request for pid " << crash_pid; } else { perform_request(std::move(crash)); } } static bool rename_tombstone_fd(borrowed_fd fd, borrowed_fd dirfd, const std::string& path) { // Always try to unlink the tombstone file. // linkat doesn't let us replace a file, so we need to unlink before linking // our results onto disk, and if we fail for some reason, we should delete // stale tombstones to avoid confusing inconsistency. int rc = unlinkat(dirfd.get(), path.c_str(), 0); if (rc != 0 && errno != ENOENT) { PLOG(ERROR) << "failed to unlink tombstone at " << path; return false; } std::string fd_path = StringPrintf("/proc/self/fd/%d", fd.get()); rc = linkat(AT_FDCWD, fd_path.c_str(), dirfd.get(), path.c_str(), AT_SYMLINK_FOLLOW); if (rc != 0) { PLOG(ERROR) << "failed to link tombstone at " << path; return false; } return true; } static void crash_completed(borrowed_fd sockfd, std::unique_ptr crash) { TombstonedCrashPacket request = {}; CrashQueue* queue = CrashQueue::for_crash(crash); ssize_t rc = TEMP_FAILURE_RETRY(read(sockfd.get(), &request, sizeof(request))); if (rc == -1) { PLOG(WARNING) << "failed to read from crash socket"; return; } else if (rc != sizeof(request)) { LOG(WARNING) << "crash socket received short read of length " << rc << " (expected " << sizeof(request) << ")"; return; } if (request.packet_type != CrashPacketType::kCompletedDump) { LOG(WARNING) << "unexpected crash packet type, expected kCompletedDump, received " << uint32_t(request.packet_type); return; } if (crash->output.text.fd == -1) { LOG(WARNING) << "missing output fd"; return; } CrashArtifactPaths paths = queue->get_next_artifact_paths(); if (rename_tombstone_fd(crash->output.text.fd, queue->dir_fd(), paths.text)) { if (crash->crash_type == kDebuggerdJavaBacktrace) { LOG(ERROR) << "Traces for pid " << crash->crash_pid << " written to: " << paths.text; } else { // NOTE: Several tools parse this log message to figure out where the // tombstone associated with a given native crash was written. Any changes // to this message must be carefully considered. LOG(ERROR) << "Tombstone written to: " << paths.text; } } if (crash->output.proto && crash->output.proto->fd != -1) { if (!paths.proto) { LOG(ERROR) << "missing path for proto tombstone"; } else { rename_tombstone_fd(crash->output.proto->fd, queue->dir_fd(), *paths.proto); } } // If we don't have O_TMPFILE, we need to clean up after ourselves. if (crash->output.text.temporary_path) { rc = unlinkat(queue->dir_fd().get(), crash->output.text.temporary_path->c_str(), 0); if (rc != 0) { PLOG(ERROR) << "failed to unlink temporary tombstone at " << paths.text; } } if (crash->output.proto && crash->output.proto->temporary_path) { rc = unlinkat(queue->dir_fd().get(), crash->output.proto->temporary_path->c_str(), 0); if (rc != 0) { PLOG(ERROR) << "failed to unlink temporary proto tombstone"; } } } static void crash_completed_cb(evutil_socket_t sockfd, short ev, void* arg) { std::unique_ptr crash(static_cast(arg)); CrashQueue* queue = CrashQueue::for_crash(crash); queue->on_crash_completed(); if ((ev & EV_READ) == EV_READ) { crash_completed(sockfd, std::move(crash)); } // If there's something queued up, let them proceed. queue->maybe_dequeue_crashes(perform_request); } int main(int, char* []) { umask(0117); // Don't try to connect to ourselves if we crash. struct sigaction action = {}; action.sa_handler = [](int signal) { LOG(ERROR) << "received fatal signal " << signal; _exit(1); }; debuggerd_register_handlers(&action); int intercept_socket = android_get_control_socket(kTombstonedInterceptSocketName); int crash_socket = android_get_control_socket(kTombstonedCrashSocketName); if (intercept_socket == -1 || crash_socket == -1) { PLOG(FATAL) << "failed to get socket from init"; } evutil_make_socket_nonblocking(intercept_socket); evutil_make_socket_nonblocking(crash_socket); event_base* base = event_base_new(); if (!base) { LOG(FATAL) << "failed to create event_base"; } intercept_manager = new InterceptManager(base, intercept_socket); evconnlistener* tombstone_listener = evconnlistener_new(base, crash_accept_cb, CrashQueue::for_tombstones(), LEV_OPT_CLOSE_ON_FREE, -1 /* backlog */, crash_socket); if (!tombstone_listener) { LOG(FATAL) << "failed to create evconnlistener for tombstones."; } if (kJavaTraceDumpsEnabled) { const int java_trace_socket = android_get_control_socket(kTombstonedJavaTraceSocketName); if (java_trace_socket == -1) { PLOG(FATAL) << "failed to get socket from init"; } evutil_make_socket_nonblocking(java_trace_socket); evconnlistener* java_trace_listener = evconnlistener_new(base, crash_accept_cb, CrashQueue::for_anrs(), LEV_OPT_CLOSE_ON_FREE, -1 /* backlog */, java_trace_socket); if (!java_trace_listener) { LOG(FATAL) << "failed to create evconnlistener for java traces."; } } LOG(INFO) << "tombstoned successfully initialized"; event_base_dispatch(base); }