1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 18 #define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 19 20 #include <vector> 21 22 #include "perfetto/protozero/proto_utils.h" 23 #include "perfetto/trace_processor/status.h" 24 #include "src/trace_processor/importers/common/trace_blob_view.h" 25 #include "src/trace_processor/importers/gzip/gzip_utils.h" 26 #include "src/trace_processor/util/status_macros.h" 27 28 #include "protos/perfetto/trace/trace.pbzero.h" 29 #include "protos/perfetto/trace/trace_packet.pbzero.h" 30 31 namespace perfetto { 32 namespace trace_processor { 33 34 // Reads a protobuf trace in chunks and extracts boundaries of trace packets 35 // (or subfields, for the case of ftrace) with their timestamps. 36 class ProtoTraceTokenizer { 37 public: 38 ProtoTraceTokenizer(); 39 40 template <typename Callback = util::Status(TraceBlobView)> Tokenize(std::unique_ptr<uint8_t[]> owned_buf,size_t size,Callback callback)41 util::Status Tokenize(std::unique_ptr<uint8_t[]> owned_buf, 42 size_t size, 43 Callback callback) { 44 uint8_t* data = &owned_buf[0]; 45 if (!partial_buf_.empty()) { 46 // It takes ~5 bytes for a proto preamble + the varint size. 47 const size_t kHeaderBytes = 5; 48 if (PERFETTO_UNLIKELY(partial_buf_.size() < kHeaderBytes)) { 49 size_t missing_len = std::min(kHeaderBytes - partial_buf_.size(), size); 50 partial_buf_.insert(partial_buf_.end(), &data[0], &data[missing_len]); 51 if (partial_buf_.size() < kHeaderBytes) 52 return util::OkStatus(); 53 data += missing_len; 54 size -= missing_len; 55 } 56 57 // At this point we have enough data in |partial_buf_| to read at least 58 // the field header and know the size of the next TracePacket. 59 const uint8_t* pos = &partial_buf_[0]; 60 uint8_t proto_field_tag = *pos; 61 uint64_t field_size = 0; 62 // We cannot do &partial_buf_[partial_buf_.size()] because that crashes 63 // on MSVC STL debug builds, so does &*partial_buf_.end(). 64 const uint8_t* next = protozero::proto_utils::ParseVarInt( 65 ++pos, &partial_buf_.front() + partial_buf_.size(), &field_size); 66 bool parse_failed = next == pos; 67 pos = next; 68 if (proto_field_tag != kTracePacketTag || field_size == 0 || 69 parse_failed) { 70 return util::ErrStatus( 71 "Failed parsing a TracePacket from the partial buffer"); 72 } 73 74 // At this point we know how big the TracePacket is. 75 size_t hdr_size = static_cast<size_t>(pos - &partial_buf_[0]); 76 size_t size_incl_header = static_cast<size_t>(field_size + hdr_size); 77 PERFETTO_DCHECK(size_incl_header > partial_buf_.size()); 78 79 // There is a good chance that between the |partial_buf_| and the new 80 // |data| of the current call we have enough bytes to parse a TracePacket. 81 if (partial_buf_.size() + size >= size_incl_header) { 82 // Create a new buffer for the whole TracePacket and copy into that: 83 // 1) The beginning of the TracePacket (including the proto header) from 84 // the partial buffer. 85 // 2) The rest of the TracePacket from the current |data| buffer (note 86 // that we might have consumed already a few bytes form |data| 87 // earlier in this function, hence we need to keep |off| into 88 // account). 89 std::unique_ptr<uint8_t[]> buf(new uint8_t[size_incl_header]); 90 memcpy(&buf[0], partial_buf_.data(), partial_buf_.size()); 91 // |size_missing| is the number of bytes for the rest of the TracePacket 92 // in |data|. 93 size_t size_missing = size_incl_header - partial_buf_.size(); 94 memcpy(&buf[partial_buf_.size()], &data[0], size_missing); 95 data += size_missing; 96 size -= size_missing; 97 partial_buf_.clear(); 98 uint8_t* buf_start = &buf[0]; // Note that buf is std::moved below. 99 RETURN_IF_ERROR(ParseInternal(std::move(buf), buf_start, 100 size_incl_header, callback)); 101 } else { 102 partial_buf_.insert(partial_buf_.end(), data, &data[size]); 103 return util::OkStatus(); 104 } 105 } 106 return ParseInternal(std::move(owned_buf), data, size, callback); 107 } 108 109 private: 110 static constexpr uint8_t kTracePacketTag = 111 protozero::proto_utils::MakeTagLengthDelimited( 112 protos::pbzero::Trace::kPacketFieldNumber); 113 114 template <typename Callback = util::Status(TraceBlobView)> ParseInternal(std::unique_ptr<uint8_t[]> owned_buf,uint8_t * data,size_t size,Callback callback)115 util::Status ParseInternal(std::unique_ptr<uint8_t[]> owned_buf, 116 uint8_t* data, 117 size_t size, 118 Callback callback) { 119 PERFETTO_DCHECK(data >= &owned_buf[0]); 120 const uint8_t* start = &owned_buf[0]; 121 const size_t data_off = static_cast<size_t>(data - start); 122 TraceBlobView whole_buf(std::move(owned_buf), data_off, size); 123 124 protos::pbzero::Trace::Decoder decoder(data, size); 125 for (auto it = decoder.packet(); it; ++it) { 126 protozero::ConstBytes packet = *it; 127 size_t field_offset = whole_buf.offset_of(packet.data); 128 TraceBlobView sliced = whole_buf.slice(field_offset, packet.size); 129 RETURN_IF_ERROR(ParsePacket(std::move(sliced), callback)); 130 } 131 132 const size_t bytes_left = decoder.bytes_left(); 133 if (bytes_left > 0) { 134 PERFETTO_DCHECK(partial_buf_.empty()); 135 partial_buf_.insert(partial_buf_.end(), &data[decoder.read_offset()], 136 &data[decoder.read_offset() + bytes_left]); 137 } 138 return util::OkStatus(); 139 } 140 141 template <typename Callback = util::Status(TraceBlobView)> ParsePacket(TraceBlobView packet,Callback callback)142 util::Status ParsePacket(TraceBlobView packet, Callback callback) { 143 protos::pbzero::TracePacket::Decoder decoder(packet.data(), 144 packet.length()); 145 if (decoder.has_compressed_packets()) { 146 if (!gzip::IsGzipSupported()) { 147 return util::Status( 148 "Cannot decode compressed packets. Zlib not enabled"); 149 } 150 151 protozero::ConstBytes field = decoder.compressed_packets(); 152 const size_t field_off = packet.offset_of(field.data); 153 TraceBlobView compressed_packets = packet.slice(field_off, field.size); 154 TraceBlobView packets(nullptr, 0, 0); 155 156 RETURN_IF_ERROR(Decompress(std::move(compressed_packets), &packets)); 157 158 const uint8_t* start = packets.data(); 159 const uint8_t* end = packets.data() + packets.length(); 160 const uint8_t* ptr = start; 161 while ((end - ptr) > 2) { 162 const uint8_t* packet_start = ptr; 163 if (PERFETTO_UNLIKELY(*ptr != kTracePacketTag)) 164 return util::ErrStatus("Expected TracePacket tag"); 165 uint64_t packet_size = 0; 166 ptr = protozero::proto_utils::ParseVarInt(++ptr, end, &packet_size); 167 size_t packet_offset = static_cast<size_t>(ptr - start); 168 ptr += packet_size; 169 if (PERFETTO_UNLIKELY((ptr - packet_start) < 2 || ptr > end)) 170 return util::ErrStatus("Invalid packet size"); 171 172 TraceBlobView sliced = 173 packets.slice(packet_offset, static_cast<size_t>(packet_size)); 174 RETURN_IF_ERROR(ParsePacket(std::move(sliced), callback)); 175 } 176 return util::OkStatus(); 177 } 178 return callback(std::move(packet)); 179 } 180 181 util::Status Decompress(TraceBlobView input, TraceBlobView* output); 182 183 // Used to glue together trace packets that span across two (or more) 184 // Parse() boundaries. 185 std::vector<uint8_t> partial_buf_; 186 187 // Allows support for compressed trace packets. 188 GzipDecompressor decompressor_; 189 }; 190 191 } // namespace trace_processor 192 } // namespace perfetto 193 194 #endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 195