src/quipper/huge_page_deducer.cc

#include "huge_page_deducer.h"

#include <limits>

#include "perf_data_utils.h"

#include "base/logging.h"

using PerfEvent = quipper::PerfDataProto::PerfEvent;
using MMapEvent = quipper::PerfDataProto::MMapEvent;

namespace quipper {
namespace {

const char kAnonFilename[] = "//anon";
const size_t kHugepageSize = 1 << 21;

bool IsAnon(const MMapEvent& event) {
  return event.filename() == kAnonFilename;
}

// IsContiguous returns true if mmap |a| is immediately followed by |b|
// within a process' address space.
bool IsContiguous(const MMapEvent& a, const MMapEvent& b) {
  return a.pid() == b.pid() && (a.start() + a.len()) == b.start();
}

// IsEquivalentFile returns true iff |a| and |b| have the same name, or if
// either of them are anonymous memory (and thus likely to be a --hugepage_text
// version of the same file).
bool IsEquivalentFile(const MMapEvent& a, const MMapEvent& b) {
  // perf attributes neighboring anonymous mappings under the argv[0]
  // filename rather than "//anon", so check filename equality, as well as
  // anonymous.
  return a.filename() == b.filename() || IsAnon(a) || IsAnon(b);
}

// Helper to correctly update a filename on a PerfEvent that contains an
// MMapEvent.
void SetMmapFilename(PerfEvent* event, const string& new_filename,
                     uint64_t new_filename_md5_prefix) {
  CHECK(event->has_mmap_event());
  event->mutable_header()->set_size(
      event->header().size() + GetUint64AlignedStringLength(new_filename) -
      GetUint64AlignedStringLength(event->mmap_event().filename()));

  event->mutable_mmap_event()->set_filename(new_filename);
  event->mutable_mmap_event()->set_filename_md5_prefix(new_filename_md5_prefix);
}
}  // namespace

namespace {

// MMapRange represents an index into a PerfEvents sequence that contains
// a contiguous region of mmaps that have all of the same filename and pgoff.
class MMapRange {
 public:
  // Default constructor is an invalid range.
  MMapRange()
      : first_(std::numeric_limits<int>::max()),
        last_(std::numeric_limits<int>::min()) {}

  // Construct a real range.
  MMapRange(int first_index, int last_index)
      : first_(first_index), last_(last_index) {}

  uint64 Len(const RepeatedPtrField<PerfEvent>& events) const {
    auto& first = events.Get(first_).mmap_event();
    auto& last = events.Get(last_).mmap_event();
    return last.start() - first.start() + last.len();
  }

  int FirstIndex() const { return first_; }
  int LastIndex() const { return last_; }
  bool IsValid() const { return first_ <= last_; }

  const MMapEvent& FirstMmap(const RepeatedPtrField<PerfEvent>& events) const {
    return events.Get(first_).mmap_event();
  }

  const MMapEvent& LastMmap(const RepeatedPtrField<PerfEvent>& events) const {
    return events.Get(last_).mmap_event();
  }

 private:
  int first_;
  int last_;
};

std::ostream& operator<<(std::ostream& os, const MMapRange& r) {
  os << "[" << r.FirstIndex() << "," << r.LastIndex() << "]";
  return os;
}

// MMapRange version of IsContiguous(MMapEvent, MMapEvent).
bool IsContiguous(const RepeatedPtrField<PerfEvent>& events, const MMapRange& a,
                  const MMapRange& b) {
  return IsContiguous(a.LastMmap(events), b.FirstMmap(events));
}

// MMapRange version of IsIsEquivalent(MMapEvent, MMapEvent).
bool IsEquivalentFile(const RepeatedPtrField<PerfEvent>& events,
                      const MMapRange& a, const MMapRange& b) {
  // Because a range has the same file for all mmaps within it, assume that
  // checking any mmap in |a| with any in |b| is sufficient.
  return IsEquivalentFile(a.LastMmap(events), b.FirstMmap(events));
}

// FindRange returns a MMapRange of contiguous MmapEvents that:
// - either:
//   - contains 1 or more MmapEvents with pgoff == 0
//   - is a single MmapEvent with pgoff != 0
// - and:
//   - has the same filename for all entries
// Otherwise, if none can be found, an invalid range will be returned.
MMapRange FindRange(const RepeatedPtrField<PerfEvent>& events, int start) {
  const MMapEvent* prev_mmap = nullptr;
  MMapRange range;
  for (int i = start; i < events.size(); i++) {
    const PerfEvent& event = events.Get(i);
    // Skip irrelevant events
    if (!event.has_mmap_event()) {
      continue;
    }
    // Skip dynamic mmap() events. Hugepage deduction only works on mmaps as
    // synthesized by perf from /proc/${pid}/maps, which have timestamp==0.
    // Support for deducing hugepages from a sequence of mmap()/mremap() calls
    // would require additional deduction logic.
    if (event.timestamp() != 0) {
      continue;
    }
    const MMapEvent& mmap = events.Get(i).mmap_event();
    if (prev_mmap == nullptr) {
      range = MMapRange(i, i);
      prev_mmap = &mmap;
    }
    // Ranges match exactly: //anon,//anon, or file,file; If they use different
    // names, then deduction needs to consider them independently.
    if (prev_mmap->filename() != mmap.filename()) {
      break;
    }
    // If they're not virtually contiguous, they're not a single range.
    if (start != i && !IsContiguous(*prev_mmap, mmap)) {
      break;
    }
    // If this segment has a page offset, assume that it is *not* hugepage
    // backed, and thus does not need separate deduction.
    if (mmap.pgoff() != 0) {
      break;
    }
    CHECK(mmap.pgoff() == 0 || !IsAnon(mmap))
        << "Anonymous pages can't have pgoff set";
    prev_mmap = &mmap;
    range = MMapRange(range.FirstIndex(), i);
  }
  // Range has:
  // - single file
  // - virtually contiguous
  // - either: is multiple mappings *or* has pgoff=0
  return range;
}

// FindNextRange will return the next range after the given |prev_range| if
// there is one; otherwise it will return an invalid range.
MMapRange FindNextRange(const RepeatedPtrField<PerfEvent>& events,
                        const MMapRange& prev_range) {
  MMapRange ret;
  if (prev_range.IsValid() && prev_range.LastIndex() < events.size()) {
    ret = FindRange(events, prev_range.LastIndex() + 1);
  }
  return ret;
}

// UpdateRangeFromNext will set the filename / pgoff of all mmaps within |range|
// to be pgoff-contiguous with |next_range|, and match its file information.
void UpdateRangeFromNext(const MMapRange& range, const MMapRange& next_range,
                         RepeatedPtrField<PerfEvent>* events) {
  CHECK(range.LastIndex() < events->size());
  CHECK(next_range.LastIndex() < events->size());
  const MMapEvent& src = next_range.FirstMmap(*events);
  const uint64 start_pgoff = src.pgoff() - range.Len(*events);
  uint64 pgoff = start_pgoff;
  for (int i = range.FirstIndex(); i <= range.LastIndex(); i++) {
    if (!events->Get(i).has_mmap_event()) {
      continue;
    }
    PerfEvent* event = events->Mutable(i);
    MMapEvent* mmap = event->mutable_mmap_event();

    // Replace "//anon" with a regular name if possible.
    if (IsAnon(*mmap)) {
      // ANDROID-CHANGED: protobuf-lite.
      CHECK_EQ(mmap->pgoff(), 0u) << "//anon should have offset=0 for mmap";
      //                          << event->ShortDebugString();
      SetMmapFilename(event, src.filename(), src.filename_md5_prefix());
    }

    if (mmap->pgoff() == 0) {
      mmap->set_pgoff(pgoff);
      if (src.has_maj()) {
        mmap->set_maj(src.maj());
      }
      if (src.has_min()) {
        mmap->set_min(src.min());
      }
      if (src.has_ino()) {
        mmap->set_ino(src.ino());
      }
      if (src.has_ino_generation()) {
        mmap->set_ino_generation(src.ino_generation());
      }
    }
    pgoff += mmap->len();
  }
  CHECK_EQ(pgoff, start_pgoff + range.Len(*events));
}
}  // namespace

void DeduceHugePages(RepeatedPtrField<PerfEvent>* events) {
  // |prev_range|, if IsValid(), represents the preview mmap range seen (and
  // already processed / updated).
  MMapRange prev_range;
  // |range| contains the currently-being-processed mmap range, which will have
  // its hugepages ranges deduced.
  MMapRange range = FindRange(*events, 0);
  // |next_range| contains the next range to process, possibily containing
  // pgoff != 0 or !IsAnon(filename) from which the current range can be
  // updated.
  MMapRange next_range = FindNextRange(*events, range);

  for (; range.IsValid(); prev_range = range, range = next_range,
                          next_range = FindNextRange(*events, range)) {
    const bool have_next =
        (next_range.IsValid() && IsContiguous(*events, range, next_range) &&
         IsEquivalentFile(*events, range, next_range));

    // If there's no mmap after this, then we assume that this is *not* viable
    // a hugepage_text mapping. This is true unless we're really unlucky. If:
    // - the binary is mapped such that the limit is hugepage aligned
    //   (presumably 4Ki/2Mi chance == p=0.03125)
    // - and the entire binaryis hugepage_text mapped
    if (!have_next) {
      continue;
    }

    const bool have_prev =
        (prev_range.IsValid() && IsContiguous(*events, prev_range, range) &&
         IsEquivalentFile(*events, prev_range, range) &&
         IsEquivalentFile(*events, prev_range, next_range));

    uint64 start_pgoff = 0;
    if (have_prev) {
      const auto& prev = prev_range.LastMmap(*events);
      start_pgoff = prev.pgoff() + prev.len();
    }
    const auto& next = next_range.FirstMmap(*events);
    // prev.pgoff should be valid now, so let's double-check that
    // if next has a non-zero pgoff, that {prev,curr,next} will have
    // contiguous pgoff once updated.
    if (next.pgoff() >= range.Len(*events) &&
        (next.pgoff() - range.Len(*events)) == start_pgoff) {
      UpdateRangeFromNext(range, next_range, events);
    }
  }
}

void CombineMappings(RepeatedPtrField<PerfEvent>* events) {
  // Combine mappings
  RepeatedPtrField<PerfEvent> new_events;
  new_events.Reserve(events->size());

  // |prev| is the index of the last mmap_event in |new_events| (or
  // |new_events.size()| if no mmap_events have been inserted yet).
  int prev = 0;
  for (int i = 0; i < events->size(); ++i) {
    PerfEvent* event = events->Mutable(i);
    if (!event->has_mmap_event()) {
      new_events.Add()->Swap(event);
      continue;
    }

    const MMapEvent& mmap = event->mmap_event();
    // Try to merge mmap with |new_events[prev]|.
    while (prev < new_events.size() && !new_events.Get(prev).has_mmap_event()) {
      prev++;
    }

    if (prev >= new_events.size()) {
      new_events.Add()->Swap(event);
      continue;
    }

    MMapEvent* prev_mmap = new_events.Mutable(prev)->mutable_mmap_event();

    // Don't use IsEquivalentFile(); we don't want to combine //anon with
    // files if DeduceHugepages didn't already fix up the mappings.
    const bool file_match = prev_mmap->filename() == mmap.filename();
    const bool pgoff_contiguous =
        file_match && (prev_mmap->pgoff() + prev_mmap->len() == mmap.pgoff());

    const bool combine_mappings =
        IsContiguous(*prev_mmap, mmap) && pgoff_contiguous;
    if (!combine_mappings) {
      new_events.Add()->Swap(event);
      prev++;
      continue;
    }

    // Combine the lengths of the two mappings.
    prev_mmap->set_len(prev_mmap->len() + mmap.len());
  }

  events->Swap(&new_events);
}

}  // namespace quipper