1import re, os
2from autotest_lib.tko import utils as tko_utils, models, status_lib
3from autotest_lib.tko.parsers import base
4
5class NoHostnameError(Exception):
6    pass
7
8
9class BoardLabelError(Exception):
10    pass
11
12
13class job(models.job):
14    def __init__(self, dir):
15        job_dict = job.load_from_dir(dir)
16        super(job, self).__init__(dir, **job_dict)
17
18
19    @classmethod
20    def load_from_dir(cls, dir):
21        keyval = cls.read_keyval(dir)
22        tko_utils.dprint(str(keyval))
23
24        user = keyval.get("user", None)
25        label = keyval.get("label", None)
26        queued_time = tko_utils.get_timestamp(keyval, "job_queued")
27        started_time = tko_utils.get_timestamp(keyval, "job_started")
28        finished_time = tko_utils.get_timestamp(keyval, "job_finished")
29        machine = cls.determine_hostname(keyval, dir)
30        machine_group = cls.determine_machine_group(machine, dir)
31        machine_owner = keyval.get("owner", None)
32
33        aborted_by = keyval.get("aborted_by", None)
34        aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
35
36        return {"user": user, "label": label, "machine": machine,
37                "queued_time": queued_time, "started_time": started_time,
38                "finished_time": finished_time, "machine_owner": machine_owner,
39                "machine_group": machine_group, "aborted_by": aborted_by,
40                "aborted_on": aborted_at, "keyval_dict": keyval}
41
42
43    @classmethod
44    def determine_hostname(cls, keyval, job_dir):
45        host_group_name = keyval.get("host_group_name", None)
46        machine = keyval.get("hostname", "")
47        is_multimachine = "," in machine
48
49        # determine what hostname to use
50        if host_group_name:
51            if is_multimachine or not machine:
52                tko_utils.dprint("Using host_group_name %r instead of "
53                                 "machine name." % host_group_name)
54                machine = host_group_name
55        elif is_multimachine:
56            try:
57                machine = job.find_hostname(job_dir) # find a unique hostname
58            except NoHostnameError:
59                pass  # just use the comma-separated name
60
61        tko_utils.dprint("MACHINE NAME: %s" % machine)
62        return machine
63
64
65    @classmethod
66    def determine_machine_group(cls, hostname, job_dir):
67        machine_groups = set()
68        for individual_hostname in hostname.split(","):
69            host_keyval = models.test.parse_host_keyval(job_dir,
70                                                        individual_hostname)
71            if not host_keyval:
72                tko_utils.dprint('Unable to parse host keyval for %s'
73                                 % individual_hostname)
74            elif 'labels' in host_keyval:
75                # Use board label as machine group. This is to avoid the
76                # confusion of multiple boards mapping to the same platform in
77                # wmatrix. With this change, wmatrix will group tests with the
78                # same board, rather than the same platform.
79                labels = host_keyval['labels'].split(',')
80                board_labels = [l[8:] for l in labels
81                               if l.startswith('board%3A')]
82                if board_labels:
83                    # Testbeds have multiple boards so concat them into a
84                    # single string then add it to the machine_groups list.
85                    machine_groups.add(','.join(board_labels))
86                else:
87                    error = ('Failed to retrieve board label from host labels: '
88                             '%s' % host_keyval['labels'])
89                    tko_utils.dprint(error)
90                    raise BoardLabelError(error)
91            elif "platform" in host_keyval:
92                machine_groups.add(host_keyval["platform"])
93        machine_group = ",".join(sorted(machine_groups))
94        tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
95        return machine_group
96
97
98    @staticmethod
99    def find_hostname(path):
100        hostname = os.path.join(path, "sysinfo", "hostname")
101        try:
102            machine = open(hostname).readline().rstrip()
103            return machine
104        except Exception:
105            tko_utils.dprint("Could not read a hostname from "
106                             "sysinfo/hostname")
107
108        uname = os.path.join(path, "sysinfo", "uname_-a")
109        try:
110            machine = open(uname).readline().split()[1]
111            return machine
112        except Exception:
113            tko_utils.dprint("Could not read a hostname from "
114                             "sysinfo/uname_-a")
115
116        raise NoHostnameError("Unable to find a machine name")
117
118
119class kernel(models.kernel):
120    def __init__(self, job, verify_ident=None):
121        kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
122        super(kernel, self).__init__(**kernel_dict)
123
124
125    @staticmethod
126    def load_from_dir(dir, verify_ident=None):
127        # try and load the booted kernel version
128        attributes = False
129        i = 1
130        build_dir = os.path.join(dir, "build")
131        while True:
132            if not os.path.exists(build_dir):
133                break
134            build_log = os.path.join(build_dir, "debug", "build_log")
135            attributes = kernel.load_from_build_log(build_log)
136            if attributes:
137                break
138            i += 1
139            build_dir = os.path.join(dir, "build.%d" % (i))
140
141        if not attributes:
142            if verify_ident:
143                base = verify_ident
144            else:
145                base = kernel.load_from_sysinfo(dir)
146            patches = []
147            hashes = []
148        else:
149            base, patches, hashes = attributes
150        tko_utils.dprint("kernel.__init__() found kernel version %s"
151                         % base)
152
153        # compute the kernel hash
154        if base == "UNKNOWN":
155            kernel_hash = "UNKNOWN"
156        else:
157            kernel_hash = kernel.compute_hash(base, hashes)
158
159        return {"base": base, "patches": patches,
160                "kernel_hash": kernel_hash}
161
162
163    @staticmethod
164    def load_from_sysinfo(path):
165        for subdir in ("reboot1", ""):
166            uname_path = os.path.join(path, "sysinfo", subdir,
167                                      "uname_-a")
168            if not os.path.exists(uname_path):
169                continue
170            uname = open(uname_path).readline().split()
171            return re.sub("-autotest$", "", uname[2])
172        return "UNKNOWN"
173
174
175    @staticmethod
176    def load_from_build_log(path):
177        if not os.path.exists(path):
178            return None
179
180        base, patches, hashes = "UNKNOWN", [], []
181        for line in file(path):
182            head, rest = line.split(": ", 1)
183            rest = rest.split()
184            if head == "BASE":
185                base = rest[0]
186            elif head == "PATCH":
187                patches.append(patch(*rest))
188                hashes.append(rest[2])
189        return base, patches, hashes
190
191
192class test(models.test):
193    def __init__(self, subdir, testname, status, reason, test_kernel,
194                 machine, started_time, finished_time, iterations,
195                 attributes, labels):
196        # for backwards compatibility with the original parser
197        # implementation, if there is no test version we need a NULL
198        # value to be used; also, if there is a version it should
199        # be terminated by a newline
200        if "version" in attributes:
201            attributes["version"] = str(attributes["version"])
202        else:
203            attributes["version"] = None
204
205        super(test, self).__init__(subdir, testname, status, reason,
206                                   test_kernel, machine, started_time,
207                                   finished_time, iterations,
208                                   attributes, labels)
209
210
211    @staticmethod
212    def load_iterations(keyval_path):
213        return iteration.load_from_keyval(keyval_path)
214
215
216class patch(models.patch):
217    def __init__(self, spec, reference, hash):
218        tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
219        super(patch, self).__init__(spec, reference, hash)
220        self.spec = spec
221        self.reference = reference
222        self.hash = hash
223
224
225class iteration(models.iteration):
226    @staticmethod
227    def parse_line_into_dicts(line, attr_dict, perf_dict):
228        key, value = line.split("=", 1)
229        perf_dict[key] = value
230
231
232class status_line(object):
233    def __init__(self, indent, status, subdir, testname, reason,
234                 optional_fields):
235        # pull out the type & status of the line
236        if status == "START":
237            self.type = "START"
238            self.status = None
239        elif status.startswith("END "):
240            self.type = "END"
241            self.status = status[4:]
242        else:
243            self.type = "STATUS"
244            self.status = status
245        assert (self.status is None or
246                self.status in status_lib.statuses)
247
248        # save all the other parameters
249        self.indent = indent
250        self.subdir = self.parse_name(subdir)
251        self.testname = self.parse_name(testname)
252        self.reason = reason
253        self.optional_fields = optional_fields
254
255
256    @staticmethod
257    def parse_name(name):
258        if name == "----":
259            return None
260        return name
261
262
263    @staticmethod
264    def is_status_line(line):
265        return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None
266
267
268    @classmethod
269    def parse_line(cls, line):
270        if not status_line.is_status_line(line):
271            return None
272        match = re.search(r"^(\t*)(.*)$", line, flags=re.DOTALL)
273        if not match:
274            # A more useful error message than:
275            #  AttributeError: 'NoneType' object has no attribute 'groups'
276            # to help us debug WTF happens on occasion here.
277            raise RuntimeError("line %r could not be parsed." % line)
278        indent, line = match.groups()
279        indent = len(indent)
280
281        # split the line into the fixed and optional fields
282        parts = line.rstrip("\n").split("\t")
283
284        part_index = 3
285        status, subdir, testname = parts[0:part_index]
286
287        # all optional parts should be of the form "key=value". once we've found
288        # a non-matching part, treat it and the rest of the parts as the reason.
289        optional_fields = {}
290        while part_index < len(parts):
291            kv = re.search(r"^(\w+)=(.+)", parts[part_index])
292            if not kv:
293                break
294
295            optional_fields[kv.group(1)] = kv.group(2)
296            part_index += 1
297
298        reason = "\t".join(parts[part_index:])
299
300        # build up a new status_line and return it
301        return cls(indent, status, subdir, testname, reason,
302                   optional_fields)
303
304
305class parser(base.parser):
306    @staticmethod
307    def make_job(dir):
308        return job(dir)
309
310
311    def state_iterator(self, buffer):
312        new_tests = []
313        boot_count = 0
314        group_subdir = None
315        sought_level = 0
316        stack = status_lib.status_stack()
317        current_kernel = kernel(self.job)
318        boot_in_progress = False
319        alert_pending = None
320        started_time = None
321
322        while not self.finished or buffer.size():
323            # stop processing once the buffer is empty
324            if buffer.size() == 0:
325                yield new_tests
326                new_tests = []
327                continue
328
329            # parse the next line
330            line = buffer.get()
331            tko_utils.dprint('\nSTATUS: ' + line.strip())
332            line = status_line.parse_line(line)
333            if line is None:
334                tko_utils.dprint('non-status line, ignoring')
335                continue # ignore non-status lines
336
337            # have we hit the job start line?
338            if (line.type == "START" and not line.subdir and
339                not line.testname):
340                sought_level = 1
341                tko_utils.dprint("found job level start "
342                                 "marker, looking for level "
343                                 "1 groups now")
344                continue
345
346            # have we hit the job end line?
347            if (line.type == "END" and not line.subdir and
348                not line.testname):
349                tko_utils.dprint("found job level end "
350                                 "marker, looking for level "
351                                 "0 lines now")
352                sought_level = 0
353
354            # START line, just push another layer on to the stack
355            # and grab the start time if this is at the job level
356            # we're currently seeking
357            if line.type == "START":
358                group_subdir = None
359                stack.start()
360                if line.indent == sought_level:
361                    started_time = \
362                                 tko_utils.get_timestamp(
363                        line.optional_fields, "timestamp")
364                tko_utils.dprint("start line, ignoring")
365                continue
366            # otherwise, update the status on the stack
367            else:
368                tko_utils.dprint("GROPE_STATUS: %s" %
369                                 [stack.current_status(),
370                                  line.status, line.subdir,
371                                  line.testname, line.reason])
372                stack.update(line.status)
373
374            if line.status == "ALERT":
375                tko_utils.dprint("job level alert, recording")
376                alert_pending = line.reason
377                continue
378
379            # ignore Autotest.install => GOOD lines
380            if (line.testname == "Autotest.install" and
381                line.status == "GOOD"):
382                tko_utils.dprint("Successful Autotest "
383                                 "install, ignoring")
384                continue
385
386            # ignore END lines for a reboot group
387            if (line.testname == "reboot" and line.type == "END"):
388                tko_utils.dprint("reboot group, ignoring")
389                continue
390
391            # convert job-level ABORTs into a 'CLIENT_JOB' test, and
392            # ignore other job-level events
393            if line.testname is None:
394                if (line.status == "ABORT" and
395                    line.type != "END"):
396                    line.testname = "CLIENT_JOB"
397                else:
398                    tko_utils.dprint("job level event, "
399                                    "ignoring")
400                    continue
401
402            # use the group subdir for END lines
403            if line.type == "END":
404                line.subdir = group_subdir
405
406            # are we inside a block group?
407            if (line.indent != sought_level and
408                line.status != "ABORT" and
409                not line.testname.startswith('reboot.')):
410                if line.subdir:
411                    tko_utils.dprint("set group_subdir: "
412                                     + line.subdir)
413                    group_subdir = line.subdir
414                tko_utils.dprint("ignoring incorrect indent "
415                                 "level %d != %d," %
416                                 (line.indent, sought_level))
417                continue
418
419            # use the subdir as the testname, except for
420            # boot.* and kernel.* tests
421            if (line.testname is None or
422                not re.search(r"^(boot(\.\d+)?$|kernel\.)",
423                              line.testname)):
424                if line.subdir and '.' in line.subdir:
425                    line.testname = line.subdir
426
427            # has a reboot started?
428            if line.testname == "reboot.start":
429                started_time = tko_utils.get_timestamp(
430                    line.optional_fields, "timestamp")
431                tko_utils.dprint("reboot start event, "
432                                 "ignoring")
433                boot_in_progress = True
434                continue
435
436            # has a reboot finished?
437            if line.testname == "reboot.verify":
438                line.testname = "boot.%d" % boot_count
439                tko_utils.dprint("reboot verified")
440                boot_in_progress = False
441                verify_ident = line.reason.strip()
442                current_kernel = kernel(self.job, verify_ident)
443                boot_count += 1
444
445            if alert_pending:
446                line.status = "ALERT"
447                line.reason = alert_pending
448                alert_pending = None
449
450            # create the actual test object
451            finished_time = tko_utils.get_timestamp(
452                line.optional_fields, "timestamp")
453            final_status = stack.end()
454            tko_utils.dprint("Adding: "
455                             "%s\nSubdir:%s\nTestname:%s\n%s" %
456                             (final_status, line.subdir,
457                              line.testname, line.reason))
458            new_test = test.parse_test(self.job, line.subdir,
459                                       line.testname,
460                                       final_status, line.reason,
461                                       current_kernel,
462                                       started_time,
463                                       finished_time)
464            started_time = None
465            new_tests.append(new_test)
466
467        # the job is finished, but we never came back from reboot
468        if boot_in_progress:
469            testname = "boot.%d" % boot_count
470            reason = "machine did not return from reboot"
471            tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
472                              "Testname:%s\n%s")
473                             % (testname, reason))
474            new_test = test.parse_test(self.job, None, testname,
475                                       "ABORT", reason,
476                                       current_kernel, None, None)
477            new_tests.append(new_test)
478        yield new_tests
479