1# pylint: disable=missing-docstring
2import os
3import re
4
5import common
6from autotest_lib.tko import models
7from autotest_lib.tko import status_lib
8from autotest_lib.tko import utils as tko_utils
9from autotest_lib.tko.parsers import base
10
11class NoHostnameError(Exception):
12    pass
13
14
15class BoardLabelError(Exception):
16    pass
17
18
19class job(models.job):
20    def __init__(self, dir):
21        job_dict = job.load_from_dir(dir)
22        super(job, self).__init__(dir, **job_dict)
23
24
25    @classmethod
26    def load_from_dir(cls, dir):
27        keyval = cls.read_keyval(dir)
28        tko_utils.dprint(str(keyval))
29
30        user = keyval.get("user", None)
31        label = keyval.get("label", None)
32        queued_time = tko_utils.get_timestamp(keyval, "job_queued")
33        started_time = tko_utils.get_timestamp(keyval, "job_started")
34        finished_time = tko_utils.get_timestamp(keyval, "job_finished")
35        machine = cls.determine_hostname(keyval, dir)
36        machine_group = cls.determine_machine_group(machine, dir)
37        machine_owner = keyval.get("owner", None)
38
39        aborted_by = keyval.get("aborted_by", None)
40        aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
41
42        return {"user": user, "label": label, "machine": machine,
43                "queued_time": queued_time, "started_time": started_time,
44                "finished_time": finished_time, "machine_owner": machine_owner,
45                "machine_group": machine_group, "aborted_by": aborted_by,
46                "aborted_on": aborted_at, "keyval_dict": keyval}
47
48
49    @classmethod
50    def determine_hostname(cls, keyval, job_dir):
51        host_group_name = keyval.get("host_group_name", None)
52        machine = keyval.get("hostname", "")
53        is_multimachine = "," in machine
54
55        # determine what hostname to use
56        if host_group_name:
57            if is_multimachine or not machine:
58                tko_utils.dprint("Using host_group_name %r instead of "
59                                 "machine name." % host_group_name)
60                machine = host_group_name
61        elif is_multimachine:
62            try:
63                machine = job.find_hostname(job_dir) # find a unique hostname
64            except NoHostnameError:
65                pass  # just use the comma-separated name
66
67        tko_utils.dprint("MACHINE NAME: %s" % machine)
68        return machine
69
70
71    @classmethod
72    def determine_machine_group(cls, hostname, job_dir):
73        machine_groups = set()
74        for individual_hostname in hostname.split(","):
75            host_keyval = models.test.parse_host_keyval(job_dir,
76                                                        individual_hostname)
77            if not host_keyval:
78                tko_utils.dprint('Unable to parse host keyval for %s'
79                                 % individual_hostname)
80            elif 'labels' in host_keyval:
81                # Use `model` label as machine group. This is to avoid the
82                # confusion of multiple boards mapping to the same platform in
83                # wmatrix. With this change, wmatrix will group tests with the
84                # same model, rather than the same platform.
85                labels = host_keyval['labels'].split(',')
86                board_labels = [l[8:] for l in labels
87                               if l.startswith('model%3A')]
88                # If the host doesn't have `model:` label, fall back to `board:`
89                # label.
90                if not board_labels:
91                    board_labels = [l[8:] for l in labels
92                               if l.startswith('board%3A')]
93                if board_labels:
94                    # Multiple board/model labels aren't supposed to
95                    # happen, but let's report something sane rather
96                    # than just failing.
97                    machine_groups.add(','.join(board_labels))
98                else:
99                    error = ('Failed to retrieve board label from host labels: '
100                             '%s' % host_keyval['labels'])
101                    tko_utils.dprint(error)
102                    raise BoardLabelError(error)
103            elif "platform" in host_keyval:
104                machine_groups.add(host_keyval["platform"])
105        machine_group = ",".join(sorted(machine_groups))
106        tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
107        return machine_group
108
109
110    @staticmethod
111    def find_hostname(path):
112        hostname = os.path.join(path, "sysinfo", "hostname")
113        try:
114            machine = open(hostname).readline().rstrip()
115            return machine
116        except Exception:
117            tko_utils.dprint("Could not read a hostname from "
118                             "sysinfo/hostname")
119
120        uname = os.path.join(path, "sysinfo", "uname_-a")
121        try:
122            machine = open(uname).readline().split()[1]
123            return machine
124        except Exception:
125            tko_utils.dprint("Could not read a hostname from "
126                             "sysinfo/uname_-a")
127
128        raise NoHostnameError("Unable to find a machine name")
129
130
131class kernel(models.kernel):
132    def __init__(self, job, verify_ident=None):
133        kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
134        super(kernel, self).__init__(**kernel_dict)
135
136
137    @staticmethod
138    def load_from_dir(dir, verify_ident=None):
139        # try and load the booted kernel version
140        attributes = False
141        i = 1
142        build_dir = os.path.join(dir, "build")
143        while True:
144            if not os.path.exists(build_dir):
145                break
146            build_log = os.path.join(build_dir, "debug", "build_log")
147            attributes = kernel.load_from_build_log(build_log)
148            if attributes:
149                break
150            i += 1
151            build_dir = os.path.join(dir, "build.%d" % (i))
152
153        if not attributes:
154            if verify_ident:
155                base = verify_ident
156            else:
157                base = kernel.load_from_sysinfo(dir)
158            patches = []
159            hashes = []
160        else:
161            base, patches, hashes = attributes
162        tko_utils.dprint("kernel.__init__() found kernel version %s"
163                         % base)
164
165        # compute the kernel hash
166        if base == "UNKNOWN":
167            kernel_hash = "UNKNOWN"
168        else:
169            kernel_hash = kernel.compute_hash(base, hashes)
170
171        return {"base": base, "patches": patches,
172                "kernel_hash": kernel_hash}
173
174
175    @staticmethod
176    def load_from_sysinfo(path):
177        for subdir in ("reboot1", ""):
178            uname_path = os.path.join(path, "sysinfo", subdir,
179                                      "uname_-a")
180            if not os.path.exists(uname_path):
181                continue
182            uname = open(uname_path).readline().split()
183            return re.sub("-autotest$", "", uname[2])
184        return "UNKNOWN"
185
186
187    @staticmethod
188    def load_from_build_log(path):
189        if not os.path.exists(path):
190            return None
191
192        base, patches, hashes = "UNKNOWN", [], []
193        for line in file(path):
194            head, rest = line.split(": ", 1)
195            rest = rest.split()
196            if head == "BASE":
197                base = rest[0]
198            elif head == "PATCH":
199                patches.append(patch(*rest))
200                hashes.append(rest[2])
201        return base, patches, hashes
202
203
204class test(models.test):
205    def __init__(self, subdir, testname, status, reason, test_kernel,
206                 machine, started_time, finished_time, iterations,
207                 attributes, labels):
208        # for backwards compatibility with the original parser
209        # implementation, if there is no test version we need a NULL
210        # value to be used; also, if there is a version it should
211        # be terminated by a newline
212        if "version" in attributes:
213            attributes["version"] = str(attributes["version"])
214        else:
215            attributes["version"] = None
216
217        super(test, self).__init__(subdir, testname, status, reason,
218                                   test_kernel, machine, started_time,
219                                   finished_time, iterations,
220                                   attributes, labels)
221
222
223    @staticmethod
224    def load_iterations(keyval_path):
225        return iteration.load_from_keyval(keyval_path)
226
227
228class patch(models.patch):
229    def __init__(self, spec, reference, hash):
230        tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
231        super(patch, self).__init__(spec, reference, hash)
232        self.spec = spec
233        self.reference = reference
234        self.hash = hash
235
236
237class iteration(models.iteration):
238    @staticmethod
239    def parse_line_into_dicts(line, attr_dict, perf_dict):
240        key, value = line.split("=", 1)
241        perf_dict[key] = value
242
243
244class status_line(object):
245    def __init__(self, indent, status, subdir, testname, reason,
246                 optional_fields):
247        # pull out the type & status of the line
248        if status == "START":
249            self.type = "START"
250            self.status = None
251        elif status.startswith("END "):
252            self.type = "END"
253            self.status = status[4:]
254        else:
255            self.type = "STATUS"
256            self.status = status
257        assert (self.status is None or
258                self.status in status_lib.statuses)
259
260        # save all the other parameters
261        self.indent = indent
262        self.subdir = self.parse_name(subdir)
263        self.testname = self.parse_name(testname)
264        self.reason = reason
265        self.optional_fields = optional_fields
266
267
268    @staticmethod
269    def parse_name(name):
270        if name == "----":
271            return None
272        return name
273
274
275    @staticmethod
276    def is_status_line(line):
277        return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None
278
279
280    @classmethod
281    def parse_line(cls, line):
282        if not status_line.is_status_line(line):
283            return None
284        match = re.search(r"^(\t*)(.*)$", line, flags=re.DOTALL)
285        if not match:
286            # A more useful error message than:
287            #  AttributeError: 'NoneType' object has no attribute 'groups'
288            # to help us debug WTF happens on occasion here.
289            raise RuntimeError("line %r could not be parsed." % line)
290        indent, line = match.groups()
291        indent = len(indent)
292
293        # split the line into the fixed and optional fields
294        parts = line.rstrip("\n").split("\t")
295
296        part_index = 3
297        status, subdir, testname = parts[0:part_index]
298
299        # all optional parts should be of the form "key=value". once we've found
300        # a non-matching part, treat it and the rest of the parts as the reason.
301        optional_fields = {}
302        while part_index < len(parts):
303            kv = re.search(r"^(\w+)=(.+)", parts[part_index])
304            if not kv:
305                break
306
307            optional_fields[kv.group(1)] = kv.group(2)
308            part_index += 1
309
310        reason = "\t".join(parts[part_index:])
311
312        # build up a new status_line and return it
313        return cls(indent, status, subdir, testname, reason,
314                   optional_fields)
315
316
317class parser(base.parser):
318    @staticmethod
319    def make_job(dir):
320        return job(dir)
321
322
323    def state_iterator(self, buffer):
324        new_tests = []
325        boot_count = 0
326        group_subdir = None
327        sought_level = 0
328        stack = status_lib.status_stack()
329        current_kernel = kernel(self.job)
330        boot_in_progress = False
331        alert_pending = None
332        started_time = None
333
334        while not self.finished or buffer.size():
335            # stop processing once the buffer is empty
336            if buffer.size() == 0:
337                yield new_tests
338                new_tests = []
339                continue
340
341            # parse the next line
342            line = buffer.get()
343            tko_utils.dprint('\nSTATUS: ' + line.strip())
344            line = status_line.parse_line(line)
345            if line is None:
346                tko_utils.dprint('non-status line, ignoring')
347                continue # ignore non-status lines
348
349            # have we hit the job start line?
350            if (line.type == "START" and not line.subdir and
351                not line.testname):
352                sought_level = 1
353                tko_utils.dprint("found job level start "
354                                 "marker, looking for level "
355                                 "1 groups now")
356                continue
357
358            # have we hit the job end line?
359            if (line.type == "END" and not line.subdir and
360                not line.testname):
361                tko_utils.dprint("found job level end "
362                                 "marker, looking for level "
363                                 "0 lines now")
364                sought_level = 0
365
366            # START line, just push another layer on to the stack
367            # and grab the start time if this is at the job level
368            # we're currently seeking
369            if line.type == "START":
370                group_subdir = None
371                stack.start()
372                if line.indent == sought_level:
373                    started_time = \
374                                 tko_utils.get_timestamp(
375                        line.optional_fields, "timestamp")
376                tko_utils.dprint("start line, ignoring")
377                continue
378            # otherwise, update the status on the stack
379            else:
380                tko_utils.dprint("GROPE_STATUS: %s" %
381                                 [stack.current_status(),
382                                  line.status, line.subdir,
383                                  line.testname, line.reason])
384                stack.update(line.status)
385
386            if line.status == "ALERT":
387                tko_utils.dprint("job level alert, recording")
388                alert_pending = line.reason
389                continue
390
391            # ignore Autotest.install => GOOD lines
392            if (line.testname == "Autotest.install" and
393                line.status == "GOOD"):
394                tko_utils.dprint("Successful Autotest "
395                                 "install, ignoring")
396                continue
397
398            # ignore END lines for a reboot group
399            if (line.testname == "reboot" and line.type == "END"):
400                tko_utils.dprint("reboot group, ignoring")
401                continue
402
403            # convert job-level ABORTs into a 'CLIENT_JOB' test, and
404            # ignore other job-level events
405            if line.testname is None:
406                if (line.status == "ABORT" and
407                    line.type != "END"):
408                    line.testname = "CLIENT_JOB"
409                else:
410                    tko_utils.dprint("job level event, "
411                                    "ignoring")
412                    continue
413
414            # use the group subdir for END lines
415            if line.type == "END":
416                line.subdir = group_subdir
417
418            # are we inside a block group?
419            if (line.indent != sought_level and
420                line.status != "ABORT" and
421                not line.testname.startswith('reboot.')):
422                if line.subdir:
423                    tko_utils.dprint("set group_subdir: "
424                                     + line.subdir)
425                    group_subdir = line.subdir
426                tko_utils.dprint("ignoring incorrect indent "
427                                 "level %d != %d," %
428                                 (line.indent, sought_level))
429                continue
430
431            # use the subdir as the testname, except for
432            # boot.* and kernel.* tests
433            if (line.testname is None or
434                not re.search(r"^(boot(\.\d+)?$|kernel\.)",
435                              line.testname)):
436                if line.subdir and '.' in line.subdir:
437                    line.testname = line.subdir
438
439            # has a reboot started?
440            if line.testname == "reboot.start":
441                started_time = tko_utils.get_timestamp(
442                    line.optional_fields, "timestamp")
443                tko_utils.dprint("reboot start event, "
444                                 "ignoring")
445                boot_in_progress = True
446                continue
447
448            # has a reboot finished?
449            if line.testname == "reboot.verify":
450                line.testname = "boot.%d" % boot_count
451                tko_utils.dprint("reboot verified")
452                boot_in_progress = False
453                verify_ident = line.reason.strip()
454                current_kernel = kernel(self.job, verify_ident)
455                boot_count += 1
456
457            if alert_pending:
458                line.status = "ALERT"
459                line.reason = alert_pending
460                alert_pending = None
461
462            # create the actual test object
463            finished_time = tko_utils.get_timestamp(
464                line.optional_fields, "timestamp")
465            final_status = stack.end()
466            tko_utils.dprint("Adding: "
467                             "%s\nSubdir:%s\nTestname:%s\n%s" %
468                             (final_status, line.subdir,
469                              line.testname, line.reason))
470            new_test = test.parse_test(self.job, line.subdir,
471                                       line.testname,
472                                       final_status, line.reason,
473                                       current_kernel,
474                                       started_time,
475                                       finished_time)
476            started_time = None
477            new_tests.append(new_test)
478
479        # the job is finished, but we never came back from reboot
480        if boot_in_progress:
481            testname = "boot.%d" % boot_count
482            reason = "machine did not return from reboot"
483            tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
484                              "Testname:%s\n%s")
485                             % (testname, reason))
486            new_test = test.parse_test(self.job, None, testname,
487                                       "ABORT", reason,
488                                       current_kernel, None, None)
489            new_tests.append(new_test)
490        yield new_tests
491