1#!/usr/bin/env python3
2# Copyright 2020 The Pigweed Authors
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may not
5# use this file except in compliance with the License. You may obtain a copy of
6# the License at
7#
8#     https://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations under
14# the License.
15r"""
16Generates json trace files viewable using chrome://tracing from binary
17trace files.
18
19Example usage:
20python pw_trace_tokenized/py/trace_tokenized.py -i trace.bin -o trace.json
21./out/host_clang_debug/obj/pw_trace_tokenized/bin/trace_tokenized_example_basic
22"""
23from enum import IntEnum
24import argparse
25import logging
26import struct
27import sys
28from pw_tokenizer import database, tokens
29from pw_trace import trace
30
31_LOG = logging.getLogger('pw_trace_tokenizer')
32
33
34def varint_decode(encoded):
35    # Taken from pw_tokenizer.decode._decode_signed_integer
36    count = 0
37    result = 0
38    shift = 0
39    for byte in encoded:
40        count += 1
41        result |= (byte & 0x7f) << shift
42        if not byte & 0x80:
43            return result, count
44
45        shift += 7
46        if shift >= 64:
47            break  # Error
48    return None
49
50
51# Token string: "event_type|flag|module|group|label|<optional data_fmt>"
52class TokenIdx(IntEnum):
53    EventType = 0
54    Flag = 1
55    Module = 2
56    Group = 3
57    Label = 4
58    data_fmt = 5  # optional
59
60
61def get_trace_type(type_str):
62    if type_str == "PW_TRACE_EVENT_TYPE_INSTANT":
63        return trace.TraceType.Instantaneous
64    if type_str == "PW_TRACE_EVENT_TYPE_INSTANT_GROUP":
65        return trace.TraceType.InstantaneousGroup
66    if type_str == "PW_TRACE_EVENT_TYPE_ASYNC_START":
67        return trace.TraceType.AsyncStart
68    if type_str == "PW_TRACE_EVENT_TYPE_ASYNC_STEP":
69        return trace.TraceType.AsyncStep
70    if type_str == "PW_TRACE_EVENT_TYPE_ASYNC_END":
71        return trace.TraceType.AsyncEnd
72    if type_str == "PW_TRACE_EVENT_TYPE_DURATION_START":
73        return trace.TraceType.DurationStart
74    if type_str == "PW_TRACE_EVENT_TYPE_DURATION_END":
75        return trace.TraceType.DurationEnd
76    if type_str == "PW_TRACE_EVENT_TYPE_DURATION_GROUP_START":
77        return trace.TraceType.DurationGroupStart
78    if type_str == "PW_TRACE_EVENT_TYPE_DURATION_GROUP_END":
79        return trace.TraceType.DurationGroupEnd
80    return trace.TraceType.Invalid
81
82
83def has_trace_id(token_string):
84    token_values = token_string.split("|")
85    return trace.event_has_trace_id(token_values[TokenIdx.EventType])
86
87
88def has_data(token_string):
89    token_values = token_string.split("|")
90    return len(token_values) > TokenIdx.data_fmt
91
92
93def create_trace_event(token_string, timestamp_us, trace_id, data):
94    token_values = token_string.split("|")
95    return trace.TraceEvent(event_type=get_trace_type(
96        token_values[TokenIdx.EventType]),
97                            module=token_values[TokenIdx.Module],
98                            label=token_values[TokenIdx.Label],
99                            timestamp_us=timestamp_us,
100                            group=token_values[TokenIdx.Group],
101                            trace_id=trace_id,
102                            flags=token_values[TokenIdx.Flag],
103                            has_data=has_data(token_string),
104                            data_fmt=(token_values[TokenIdx.data_fmt]
105                                      if has_data(token_string) else ""),
106                            data=data if has_data(token_string) else b'')
107
108
109def parse_trace_event(buffer, db, last_time, ticks_per_second=1000):
110    """Parse a single trace event from bytes"""
111    us_per_tick = 1000000 / ticks_per_second
112    idx = 0
113    # Read token
114    token = struct.unpack('I', buffer[idx:idx + 4])[0]
115    idx += 4
116
117    # Decode token
118    if len(db.token_to_entries[token]) == 0:
119        _LOG.error("token not found: %08x", token)
120        return None
121
122    token_string = str(db.token_to_entries[token][0])
123
124    # Read time
125    time_delta, time_bytes = varint_decode(buffer[idx:])
126    timestamp_us = last_time + us_per_tick * time_delta
127    idx += time_bytes
128
129    # Trace ID
130    trace_id = None
131    if has_trace_id(token_string) and idx < len(buffer):
132        trace_id, trace_id_bytes = varint_decode(buffer[idx:])
133        idx += trace_id_bytes
134
135    # Data
136    data = None
137    if has_data(token_string) and idx < len(buffer):
138        data = buffer[idx:]
139
140    # Create trace event
141    return create_trace_event(token_string, timestamp_us, trace_id, data)
142
143
144def get_trace_events(databases, raw_trace_data):
145    """Handles the decoding traces."""
146
147    db = tokens.Database.merged(*databases)
148    last_timestamp = 0
149    events = []
150    idx = 0
151
152    while idx + 1 < len(raw_trace_data):
153        # Read size
154        size = int(raw_trace_data[idx])
155        if idx + size > len(raw_trace_data):
156            _LOG.error("incomplete file")
157            break
158
159        event = parse_trace_event(raw_trace_data[idx + 1:idx + 1 + size], db,
160                                  last_timestamp)
161        if event:
162            last_timestamp = event.timestamp_us
163            events.append(event)
164        idx = idx + size + 1
165    return events
166
167
168def get_trace_data_from_file(input_file_name):
169    """Handles the decoding traces."""
170    with open(input_file_name, "rb") as input_file:
171        return input_file.read()
172    return None
173
174
175def save_trace_file(trace_lines, file_name):
176    """Handles generating the trace file."""
177    with open(file_name, 'w') as output_file:
178        output_file.write("[")
179        for line in trace_lines:
180            output_file.write("%s,\n" % line)
181        output_file.write("{}]")
182
183
184def get_trace_events_from_file(databases, input_file_name):
185    """Get trace events from a file."""
186    raw_trace_data = get_trace_data_from_file(input_file_name)
187    return get_trace_events(databases, raw_trace_data)
188
189
190def _parse_args():
191    """Parse and return command line arguments."""
192
193    parser = argparse.ArgumentParser(
194        description=__doc__,
195        formatter_class=argparse.RawDescriptionHelpFormatter)
196    parser.add_argument(
197        'databases',
198        nargs='+',
199        action=database.LoadTokenDatabases,
200        help='Databases (ELF, binary, or CSV) to use to lookup tokens.')
201    parser.add_argument(
202        '-i',
203        '--input',
204        dest='input_file',
205        help='The binary trace input file, generated using trace_to_file.h.')
206    parser.add_argument('-o',
207                        '--output',
208                        dest='output_file',
209                        help=('The json file to which to write the output.'))
210
211    return parser.parse_args()
212
213
214def _main(args):
215    events = get_trace_events_from_file(args.databases, args.input_file)
216    json_lines = trace.generate_trace_json(events)
217    save_trace_file(json_lines, args.output_file)
218
219
220if __name__ == '__main__':
221    if sys.version_info[0] < 3:
222        sys.exit('ERROR: The detokenizer command line tools require Python 3.')
223    _main(_parse_args())
224