1// Copyright 2017 syzkaller project authors. All rights reserved.
2// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
3
4package report
5
6import (
7	"bufio"
8	"bytes"
9	"fmt"
10	"net/mail"
11	"path/filepath"
12	"regexp"
13	"strconv"
14	"strings"
15	"time"
16
17	"github.com/google/syzkaller/pkg/osutil"
18	"github.com/google/syzkaller/pkg/symbolizer"
19)
20
21type linux struct {
22	kernelSrc             string
23	kernelObj             string
24	vmlinux               string
25	symbols               map[string][]symbolizer.Symbol
26	ignores               []*regexp.Regexp
27	consoleOutputRe       *regexp.Regexp
28	questionableRe        *regexp.Regexp
29	guiltyFileBlacklist   []*regexp.Regexp
30	reportStartIgnores    [][]byte
31	infoMessagesWithStack [][]byte
32	eoi                   []byte
33}
34
35func ctorLinux(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) {
36	vmlinux := ""
37	var symbols map[string][]symbolizer.Symbol
38	if kernelObj != "" {
39		vmlinux = filepath.Join(kernelObj, "vmlinux")
40		var err error
41		symbols, err = symbolizer.ReadSymbols(vmlinux)
42		if err != nil {
43			return nil, nil, err
44		}
45	}
46	ctx := &linux{
47		kernelSrc: kernelSrc,
48		kernelObj: kernelObj,
49		vmlinux:   vmlinux,
50		symbols:   symbols,
51		ignores:   ignores,
52	}
53	ctx.consoleOutputRe = regexp.MustCompile(`^(?:\*\* [0-9]+ printk messages dropped \*\* )?(?:.* login: )?(?:\<[0-9]+\>)?\[ *[0-9]+\.[0-9]+\] `)
54	ctx.questionableRe = regexp.MustCompile(`(?:\[\<[0-9a-f]+\>\])? \? +[a-zA-Z0-9_.]+\+0x[0-9a-f]+/[0-9a-f]+`)
55	ctx.eoi = []byte("<EOI>")
56	ctx.guiltyFileBlacklist = []*regexp.Regexp{
57		regexp.MustCompile(`.*\.h`),
58		regexp.MustCompile(`^lib/.*`),
59		regexp.MustCompile(`^virt/lib/.*`),
60		regexp.MustCompile(`^mm/kasan/.*`),
61		regexp.MustCompile(`^mm/kmsan/.*`),
62		regexp.MustCompile(`^kernel/kcov.c`),
63		regexp.MustCompile(`^mm/sl.b.c`),
64		regexp.MustCompile(`^mm/percpu.*`),
65		regexp.MustCompile(`^mm/vmalloc.c`),
66		regexp.MustCompile(`^mm/page_alloc.c`),
67		regexp.MustCompile(`^mm/util.c`),
68		regexp.MustCompile(`^kernel/rcu/.*`),
69		regexp.MustCompile(`^arch/.*/kernel/traps.c`),
70		regexp.MustCompile(`^arch/.*/mm/fault.c`),
71		regexp.MustCompile(`^kernel/locking/.*`),
72		regexp.MustCompile(`^kernel/panic.c`),
73		regexp.MustCompile(`^kernel/softirq.c`),
74		regexp.MustCompile(`^kernel/kthread.c`),
75		regexp.MustCompile(`^kernel/sched/.*.c`),
76		regexp.MustCompile(`^kernel/time/timer.c`),
77		regexp.MustCompile(`^kernel/workqueue.c`),
78		regexp.MustCompile(`^net/core/dev.c`),
79		regexp.MustCompile(`^net/core/sock.c`),
80		regexp.MustCompile(`^net/core/skbuff.c`),
81		regexp.MustCompile(`^fs/proc/generic.c`),
82	}
83	// These pattern do _not_ start a new report, i.e. can be in a middle of another report.
84	ctx.reportStartIgnores = [][]byte{
85		[]byte("invalid opcode: 0000"),
86		[]byte("Kernel panic - not syncing: panic_on_warn set"),
87		[]byte("unregister_netdevice: waiting for"),
88	}
89	// These pattern math kernel reports which are not bugs in itself but contain stack traces.
90	// If we see them in the middle of another report, we know that the report is potentially corrupted.
91	ctx.infoMessagesWithStack = [][]byte{
92		[]byte("vmalloc: allocation failure:"),
93		[]byte("FAULT_INJECTION: forcing a failure"),
94		[]byte("FAULT_FLAG_ALLOW_RETRY missing"),
95	}
96	suppressions := []string{
97		"fatal error: runtime: out of memory",
98		"fatal error: runtime: cannot allocate memory",
99		"panic: failed to start executor binary",
100		"panic: executor failed: pthread_create failed",
101		"panic: failed to create temp dir",
102		"fatal error: unexpected signal during runtime execution", // presubmably OOM turned into SIGBUS
103		"signal SIGBUS: bus error",                                // presubmably OOM turned into SIGBUS
104		"Out of memory: Kill process .* \\(syz-fuzzer\\)",
105		"Out of memory: Kill process .* \\(sshd\\)",
106		"Killed process .* \\(syz-fuzzer\\)",
107		"Killed process .* \\(sshd\\)",
108		"lowmemorykiller: Killing 'syz-fuzzer'",
109		"lowmemorykiller: Killing 'sshd'",
110		"INIT: PANIC: segmentation violation!",
111	}
112	return ctx, suppressions, nil
113}
114
115func (ctx *linux) ContainsCrash(output []byte) bool {
116	return containsCrash(output, linuxOopses, ctx.ignores)
117}
118
119func (ctx *linux) Parse(output []byte) *Report {
120	oops, startPos, endPos, logReport, consoleReport, consoleReportReliable,
121		logReportPrefix, consoleReportPrefix := ctx.parseOutput(output)
122	if oops == nil {
123		return nil
124	}
125	rep := &Report{
126		Output:   output,
127		StartPos: startPos,
128		EndPos:   endPos,
129	}
130	var report []byte
131	var reportPrefix [][]byte
132	// Try extracting report from console output only.
133	title, corrupted, format := extractDescription(consoleReportReliable, oops, linuxStackParams)
134	if title != "" {
135		report = consoleReport
136		reportPrefix = consoleReportPrefix
137	} else {
138		// Failure. Try extracting report from the whole log.
139		report = logReport
140		reportPrefix = logReportPrefix
141		title, corrupted, format = extractDescription(report, oops, linuxStackParams)
142		if title == "" {
143			panic(fmt.Sprintf("non matching oops for %q in:\n%s\n\nconsole:\n%s\n"+
144				"output [range:%v-%v]:\n%s\n",
145				oops.header, report, consoleReportReliable,
146				rep.StartPos, rep.StartPos+len(report), output))
147		}
148	}
149	rep.Title = title
150	rep.Corrupted = corrupted != ""
151	rep.CorruptedReason = corrupted
152	// Prepend 5 lines preceding start of the report,
153	// they can contain additional info related to the report.
154	for _, prefix := range reportPrefix {
155		rep.Report = append(rep.Report, prefix...)
156		rep.Report = append(rep.Report, '\n')
157	}
158	rep.Report = append(rep.Report, report...)
159	if !rep.Corrupted {
160		rep.Corrupted, rep.CorruptedReason = ctx.isCorrupted(title, report, format)
161	}
162	return rep
163}
164
165// Yes, it is complex, but all state and logic are tightly coupled. It's unclear how to simplify it.
166// nolint: gocyclo
167func (ctx *linux) parseOutput(output []byte) (
168	oops *oops, startPos, endPos int,
169	logReport, consoleReport, consoleReportReliable []byte,
170	logReportPrefix, consoleReportPrefix [][]byte) {
171	firstReportEnd := 0
172	secondReportPos := 0
173	textLines := 0
174	skipText := false
175	for pos := 0; pos < len(output); {
176		next := bytes.IndexByte(output[pos:], '\n')
177		if next != -1 {
178			next += pos
179		} else {
180			next = len(output)
181		}
182		line := output[pos:next]
183		for _, oops1 := range linuxOopses {
184			match := matchOops(line, oops1, ctx.ignores)
185			if match == -1 {
186				if oops != nil && secondReportPos == 0 {
187					for _, pattern := range ctx.infoMessagesWithStack {
188						if bytes.Contains(line, pattern) {
189							secondReportPos = pos
190							break
191						}
192					}
193				}
194				continue
195			}
196			endPos = next
197			if oops == nil {
198				oops = oops1
199				startPos = pos
200				break
201			} else if secondReportPos == 0 {
202				ignored := false
203				for _, ignore := range ctx.reportStartIgnores {
204					if bytes.Contains(line, ignore) {
205						ignored = true
206						break
207					}
208				}
209				if !ignored {
210					secondReportPos = pos
211				}
212			}
213		}
214		if oops == nil {
215			logReportPrefix = append(logReportPrefix, append([]byte{}, line...))
216			if len(logReportPrefix) > 5 {
217				logReportPrefix = logReportPrefix[1:]
218			}
219		}
220		if ctx.consoleOutputRe.Match(line) &&
221			(!ctx.questionableRe.Match(line) || bytes.Contains(line, ctx.eoi)) {
222			lineStart := bytes.Index(line, []byte("] ")) + pos + 2
223			lineEnd := next
224			if lineEnd != 0 && output[lineEnd-1] == '\r' {
225				lineEnd--
226			}
227			if oops == nil {
228				consoleReportPrefix = append(consoleReportPrefix,
229					append([]byte{}, output[lineStart:lineEnd]...))
230				if len(consoleReportPrefix) > 5 {
231					consoleReportPrefix = consoleReportPrefix[1:]
232				}
233			} else {
234				textLines++
235				ln := output[lineStart:lineEnd]
236				skipLine := skipText
237				if bytes.Contains(ln, []byte("Disabling lock debugging due to kernel taint")) {
238					skipLine = true
239				} else if textLines > 25 &&
240					bytes.Contains(ln, []byte("Kernel panic - not syncing")) {
241					// If panic_on_warn set, then we frequently have 2 stacks:
242					// one for the actual report (or maybe even more than one),
243					// and then one for panic caused by panic_on_warn. This makes
244					// reports unnecessary long and the panic (current) stack
245					// is always present in the actual report. So we strip the
246					// panic message. However, we check that we have enough lines
247					// before the panic, because sometimes we have, for example,
248					// a single WARNING line without a stack and then the panic
249					// with the stack.
250					skipText = true
251					skipLine = true
252				}
253				if !skipLine {
254					consoleReport = append(consoleReport, ln...)
255					consoleReport = append(consoleReport, '\n')
256					if secondReportPos == 0 {
257						firstReportEnd = len(consoleReport)
258					}
259				}
260			}
261		}
262		pos = next + 1
263	}
264	if oops == nil {
265		return
266	}
267	if secondReportPos == 0 {
268		secondReportPos = len(output)
269	}
270	logReport = output[startPos:secondReportPos]
271	consoleReportReliable = consoleReport[:firstReportEnd]
272	return
273}
274
275func (ctx *linux) Symbolize(rep *Report) error {
276	if ctx.vmlinux == "" {
277		return nil
278	}
279	symbolized, err := ctx.symbolize(rep.Report)
280	if err != nil {
281		return err
282	}
283	rep.Report = symbolized
284	guiltyFile := ctx.extractGuiltyFile(rep.Report)
285	if guiltyFile != "" {
286		rep.Maintainers, err = ctx.getMaintainers(guiltyFile)
287		if err != nil {
288			return err
289		}
290	}
291	return nil
292}
293
294func (ctx *linux) symbolize(text []byte) ([]byte, error) {
295	symb := symbolizer.NewSymbolizer()
296	defer symb.Close()
297	strip := ctx.stripPrefix(symb)
298	var symbolized []byte
299	s := bufio.NewScanner(bytes.NewReader(text))
300	for s.Scan() {
301		line := append([]byte{}, s.Bytes()...)
302		line = append(line, '\n')
303		line = symbolizeLine(symb.Symbolize, ctx.symbols, ctx.vmlinux, strip, line)
304		symbolized = append(symbolized, line...)
305	}
306	return symbolized, nil
307}
308
309func (ctx *linux) stripPrefix(symb *symbolizer.Symbolizer) string {
310	// Vmlinux may have been moved, so check if we can find debug info
311	// for some known functions and infer correct strip prefix from it.
312	knownSymbols := []struct {
313		symbol string
314		file   string
315	}{
316		{"__sanitizer_cov_trace_pc", "kernel/kcov.c"},
317		{"__asan_load1", "mm/kasan/kasan.c"},
318		{"start_kernel", "init/main.c"},
319	}
320	for _, s := range knownSymbols {
321		for _, covSymb := range ctx.symbols[s.symbol] {
322			frames, _ := symb.Symbolize(ctx.vmlinux, covSymb.Addr)
323			if len(frames) > 0 {
324				file := frames[len(frames)-1].File
325				if idx := strings.Index(file, s.file); idx != -1 {
326					return file[:idx]
327				}
328			}
329		}
330	}
331	// Strip vmlinux location from all paths.
332	strip, _ := filepath.Abs(ctx.vmlinux)
333	return filepath.Dir(strip) + string(filepath.Separator)
334}
335
336func symbolizeLine(symbFunc func(bin string, pc uint64) ([]symbolizer.Frame, error),
337	symbols map[string][]symbolizer.Symbol, vmlinux, strip string, line []byte) []byte {
338	match := linuxSymbolizeRe.FindSubmatchIndex(line)
339	if match == nil {
340		return line
341	}
342	fn := line[match[2]:match[3]]
343	off, err := strconv.ParseUint(string(line[match[4]:match[5]]), 16, 64)
344	if err != nil {
345		return line
346	}
347	size, err := strconv.ParseUint(string(line[match[6]:match[7]]), 16, 64)
348	if err != nil {
349		return line
350	}
351	symb := symbols[string(fn)]
352	if len(symb) == 0 {
353		return line
354	}
355	var funcStart uint64
356	for _, s := range symb {
357		if funcStart == 0 || int(size) == s.Size {
358			funcStart = s.Addr
359		}
360	}
361	frames, err := symbFunc(vmlinux, funcStart+off-1)
362	if err != nil || len(frames) == 0 {
363		return line
364	}
365	var symbolized []byte
366	for _, frame := range frames {
367		file := frame.File
368		file = strings.TrimPrefix(file, strip)
369		file = strings.TrimPrefix(file, "./")
370		info := fmt.Sprintf(" %v:%v", file, frame.Line)
371		modified := append([]byte{}, line...)
372		modified = replace(modified, match[7], match[7], []byte(info))
373		if frame.Inline {
374			end := match[7] + len(info)
375			modified = replace(modified, end, end, []byte(" [inline]"))
376			modified = replace(modified, match[2], match[7], []byte(frame.Func))
377		}
378		symbolized = append(symbolized, modified...)
379	}
380	return symbolized
381}
382
383func (ctx *linux) extractGuiltyFile(report []byte) string {
384	if linuxRcuStall.Match(report) {
385		// Special case for rcu stalls.
386		// There are too many frames that we want to skip before actual guilty frames,
387		// we would need to blacklist too many files and that would be fragile.
388		// So instead we try to extract guilty file starting from the known
389		// interrupt entry point first.
390		if pos := bytes.Index(report, []byte(" apic_timer_interrupt+0x")); pos != -1 {
391			if file := ctx.extractGuiltyFileImpl(report[pos:]); file != "" {
392				return file
393			}
394		}
395	}
396	return ctx.extractGuiltyFileImpl(report)
397}
398
399func (ctx *linux) extractGuiltyFileImpl(report []byte) string {
400	files := ctx.extractFiles(report)
401nextFile:
402	for _, file := range files {
403		for _, re := range ctx.guiltyFileBlacklist {
404			if re.MatchString(file) {
405				continue nextFile
406			}
407		}
408		return file
409	}
410	return ""
411}
412
413func (ctx *linux) getMaintainers(file string) ([]string, error) {
414	mtrs, err := ctx.getMaintainersImpl(file, false)
415	if err != nil {
416		return nil, err
417	}
418	if len(mtrs) <= 1 {
419		mtrs, err = ctx.getMaintainersImpl(file, true)
420		if err != nil {
421			return nil, err
422		}
423	}
424	return mtrs, nil
425}
426
427func (ctx *linux) getMaintainersImpl(file string, blame bool) ([]string, error) {
428	args := []string{"--no-n", "--no-rolestats"}
429	if blame {
430		args = append(args, "--git-blame")
431	}
432	args = append(args, file)
433	output, err := osutil.RunCmd(time.Minute, ctx.kernelSrc, filepath.FromSlash("scripts/get_maintainer.pl"), args...)
434	if err != nil {
435		return nil, err
436	}
437	lines := strings.Split(string(output), "\n")
438	var mtrs []string
439	for _, line := range lines {
440		addr, err := mail.ParseAddress(line)
441		if err != nil {
442			continue
443		}
444		mtrs = append(mtrs, addr.Address)
445	}
446	return mtrs, nil
447}
448
449func (ctx *linux) extractFiles(report []byte) []string {
450	matches := filenameRe.FindAll(report, -1)
451	var files []string
452	for _, match := range matches {
453		f := string(bytes.Split(match, []byte{':'})[0])
454		files = append(files, filepath.Clean(f))
455	}
456	return files
457}
458
459func (ctx *linux) isCorrupted(title string, report []byte, format oopsFormat) (bool, string) {
460	// Check if the report contains stack trace.
461	if !format.noStackTrace && !bytes.Contains(report, []byte("Call Trace")) &&
462		!bytes.Contains(report, []byte("backtrace")) {
463		return true, "no stack trace in report"
464	}
465	// Check for common title corruptions.
466	for _, re := range linuxCorruptedTitles {
467		if re.MatchString(title) {
468			return true, "title matches corrupted regexp"
469		}
470	}
471	// When a report contains 'Call Trace', 'backtrace', 'Allocated' or 'Freed' keywords,
472	// it must also contain at least a single stack frame after each of them.
473	for _, key := range linuxStackKeywords {
474		match := key.FindSubmatchIndex(report)
475		if match == nil {
476			continue
477		}
478		frames := bytes.Split(report[match[0]:], []byte{'\n'})
479		if len(frames) < 4 {
480			return true, "call trace is missed"
481		}
482		frames = frames[1:]
483		corrupted := true
484		// Check that at least one of the next few lines contains a frame.
485	outer:
486		for i := 0; i < 15 && i < len(frames); i++ {
487			for _, key1 := range linuxStackKeywords {
488				// Next stack trace starts.
489				if key1.Match(frames[i]) {
490					break outer
491				}
492			}
493			if bytes.Contains(frames[i], []byte("(stack is not available)")) ||
494				stackFrameRe.Match(frames[i]) {
495				corrupted = false
496				break
497			}
498		}
499		if corrupted {
500			return true, "no frames in a stack trace"
501		}
502	}
503	return false, ""
504}
505
506var (
507	linuxSymbolizeRe = regexp.MustCompile(`(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`)
508	stackFrameRe     = regexp.MustCompile(`^ *(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`)
509	linuxRcuStall    = compile("INFO: rcu_(?:preempt|sched|bh) (?:self-)?detected(?: expedited)? stall")
510	linuxRipFrame    = compile(`IP: (?:(?:[0-9]+:)?(?:{{PC}} +){0,2}{{FUNC}}|[0-9]+:0x[0-9a-f]+|(?:[0-9]+:)?{{PC}} +\[< *\(null\)>\] +\(null\)|[0-9]+: +\(null\))`)
511)
512
513var linuxCorruptedTitles = []*regexp.Regexp{
514	// Sometimes timestamps get merged into the middle of report description.
515	regexp.MustCompile(`\[ *[0-9]+\.[0-9]+\]`),
516}
517
518var linuxStackKeywords = []*regexp.Regexp{
519	regexp.MustCompile(`Call Trace`),
520	regexp.MustCompile(`Allocated`),
521	regexp.MustCompile(`Freed`),
522	// Match 'backtrace:', but exclude 'stack backtrace:'
523	regexp.MustCompile(`[^k] backtrace:`),
524}
525
526var linuxStackParams = &stackParams{
527	stackStartRes: linuxStackKeywords,
528	frameRes: []*regexp.Regexp{
529		compile("^ +(?:{{PC}} )?{{FUNC}}"),
530	},
531	skipPatterns: []string{
532		"__sanitizer",
533		"__asan",
534		"kasan",
535		"check_memory_region",
536		"print_address_description",
537		"panic",
538		"invalid_op",
539		"report_bug",
540		"fixup_bug",
541		"do_error",
542		"invalid_op",
543		"_trap",
544		"dump_stack",
545		"warn_slowpath",
546		"warn_alloc",
547		"__warn",
548		"debug_object",
549		"work_is_static_object",
550		"lockdep",
551		"perf_trace",
552		"lock_acquire",
553		"lock_release",
554		"register_lock_class",
555		"spin_lock",
556		"spin_unlock",
557		"raw_read_lock",
558		"raw_write_lock",
559		"down_read",
560		"down_write",
561		"down_read_trylock",
562		"down_write_trylock",
563		"up_read",
564		"up_write",
565		"mutex_lock",
566		"mutex_unlock",
567		"memcpy",
568		"memcmp",
569		"memset",
570		"strcmp",
571		"strcpy",
572		"strlen",
573		"copy_to_user",
574		"copy_from_user",
575		"put_user",
576		"get_user",
577		"might_fault",
578		"might_sleep",
579		"list_add",
580		"list_del",
581		"list_replace",
582		"list_move",
583		"list_splice",
584	},
585	corruptedLines: []*regexp.Regexp{
586		// Fault injection stacks are frequently intermixed with crash reports.
587		compile(`^ should_fail(\.[a-z]+\.[0-9]+)?\+0x`),
588		compile(`^ should_failslab(\.[a-z]+\.[0-9]+)?\+0x`),
589	},
590}
591
592func warningStackFmt(skip ...string) *stackFmt {
593	return &stackFmt{
594		// In newer kernels WARNING traps and actual stack starts after invalid_op frame,
595		// older kernels just print stack.
596		parts: []*regexp.Regexp{
597			linuxRipFrame,
598			parseStackTrace,
599		},
600		parts2: []*regexp.Regexp{
601			compile("Call Trace:"),
602			parseStackTrace,
603		},
604		skip: skip,
605	}
606}
607
608var linuxOopses = []*oops{
609	{
610		[]byte("BUG:"),
611		[]oopsFormat{
612			{
613				title:  compile("BUG: KASAN:"),
614				report: compile("BUG: KASAN: ([a-z\\-]+) in {{FUNC}}(?:.*\\n)+?.*(Read|Write) of size (?:[0-9]+)"),
615
616				fmt: "KASAN: %[1]v %[3]v in %[4]v",
617				stack: &stackFmt{
618					parts: []*regexp.Regexp{
619						compile("BUG: KASAN: (?:[a-z\\-]+) in {{FUNC}}"),
620						compile("Call Trace:"),
621						parseStackTrace,
622					},
623				},
624			},
625			{
626				title:  compile("BUG: KASAN:"),
627				report: compile("BUG: KASAN: double-free or invalid-free in {{FUNC}}"),
628				fmt:    "KASAN: invalid-free in %[2]v",
629				stack: &stackFmt{
630					parts: []*regexp.Regexp{
631						compile("BUG: KASAN: double-free or invalid-free in {{FUNC}}"),
632						compile("Call Trace:"),
633						parseStackTrace,
634					},
635					skip: []string{"kmem_", "slab_", "kfree", "vunmap", "vfree"},
636				},
637			},
638			{
639				title: compile("BUG: KASAN: ([a-z\\-]+) on address(?:.*\\n)+?.*(Read|Write) of size ([0-9]+)"),
640				fmt:   "KASAN: %[1]v %[2]v",
641			},
642			{
643				title:     compile("BUG: KASAN: (.*)"),
644				fmt:       "KASAN: %[1]v",
645				corrupted: true,
646			},
647			{
648				title: compile("BUG: KMSAN: (.*)"),
649				fmt:   "KMSAN: %[1]v",
650			},
651			{
652				title: compile("BUG: unable to handle kernel paging request"),
653				fmt:   "BUG: unable to handle kernel paging request in %[1]v",
654				stack: &stackFmt{
655					parts: []*regexp.Regexp{
656						linuxRipFrame,
657						compile("Call Trace:"),
658						parseStackTrace,
659					},
660				},
661			},
662			{
663				title: compile("BUG: unable to handle kernel NULL pointer dereference"),
664				fmt:   "BUG: unable to handle kernel NULL pointer dereference in %[1]v",
665				stack: &stackFmt{
666					parts: []*regexp.Regexp{
667						linuxRipFrame,
668						compile("Call Trace:"),
669						parseStackTrace,
670					},
671				},
672			},
673			{
674				// Sometimes with such BUG failures, the second part of the header doesn't get printed
675				// or gets corrupted, because kernel prints it as two separate printk() calls.
676				title:     compile("BUG: unable to handle kernel"),
677				fmt:       "BUG: unable to handle kernel",
678				corrupted: true,
679			},
680			{
681				title: compile("BUG: spinlock (lockup suspected|already unlocked|recursion|bad magic|wrong owner|wrong CPU)"),
682				fmt:   "BUG: spinlock %[1]v in %[2]v",
683				stack: &stackFmt{
684					parts: []*regexp.Regexp{
685						compile("Call Trace:"),
686						parseStackTrace,
687					},
688					skip: []string{"spin_"},
689				},
690			},
691			{
692				title: compile("BUG: soft lockup"),
693				fmt:   "BUG: soft lockup in %[1]v",
694				stack: &stackFmt{
695					parts: []*regexp.Regexp{
696						compile("Call Trace:"),
697						parseStackTrace,
698					},
699				},
700			},
701			{
702				title:  compile("BUG: .*still has locks held!"),
703				report: compile("BUG: .*still has locks held!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"),
704				fmt:    "BUG: still has locks held in %[1]v",
705			},
706			{
707				title:        compile("BUG: lock held when returning to user space"),
708				report:       compile("BUG: lock held when returning to user space(?:.*\\n)+?.*leaving the kernel with locks still held(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
709				fmt:          "BUG: lock held when returning to user space in %[1]v",
710				noStackTrace: true,
711			},
712			{
713				title:  compile("BUG: bad unlock balance detected!"),
714				report: compile("BUG: bad unlock balance detected!(?:.*\\n){0,15}?.*is trying to release lock(?:.*\\n){0,15}?.*{{PC}} +{{FUNC}}"),
715				fmt:    "BUG: bad unlock balance in %[1]v",
716			},
717			{
718				title:  compile("BUG: held lock freed!"),
719				report: compile("BUG: held lock freed!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"),
720				fmt:    "BUG: held lock freed in %[1]v",
721			},
722			{
723				title:        compile("BUG: Bad rss-counter state"),
724				fmt:          "BUG: Bad rss-counter state",
725				noStackTrace: true,
726			},
727			{
728				title:        compile("BUG: non-zero nr_ptes on freeing mm"),
729				fmt:          "BUG: non-zero nr_ptes on freeing mm",
730				noStackTrace: true,
731			},
732			{
733				title:        compile("BUG: non-zero nr_pmds on freeing mm"),
734				fmt:          "BUG: non-zero nr_pmds on freeing mm",
735				noStackTrace: true,
736			},
737			{
738				title: compile("BUG: Dentry .* still in use \\([0-9]+\\) \\[unmount of ([^\\]]+)\\]"),
739				fmt:   "BUG: Dentry still in use [unmount of %[1]v]",
740			},
741			{
742				title: compile("BUG: Bad page state"),
743				fmt:   "BUG: Bad page state",
744			},
745			{
746				title: compile("BUG: Bad page map"),
747				fmt:   "BUG: Bad page map",
748			},
749			{
750				title:        compile("BUG: workqueue lockup"),
751				fmt:          "BUG: workqueue lockup",
752				noStackTrace: true,
753			},
754			{
755				title: compile("BUG: sleeping function called from invalid context (.*)"),
756				fmt:   "BUG: sleeping function called from invalid context %[1]v",
757			},
758			{
759				title: compile("BUG: using __this_cpu_([a-z_]+)\\(\\) in preemptible"),
760				fmt:   "BUG: using __this_cpu_%[1]v() in preemptible code in %[2]v",
761				stack: &stackFmt{
762					parts: []*regexp.Regexp{
763						compile("Call Trace:"),
764						parseStackTrace,
765					},
766					skip: []string{"dump_stack", "preemption", "preempt"},
767				},
768			},
769			{
770				title: compile("BUG: workqueue leaked lock or atomic"),
771				report: compile("BUG: workqueue leaked lock or atomic(?:.*\\n)+?" +
772					".*last function: ([a-zA-Z0-9_]+)\\n"),
773				fmt:          "BUG: workqueue leaked lock or atomic in %[1]v",
774				noStackTrace: true,
775			},
776			{
777				title:        compile("BUG: executor-detected bug"),
778				fmt:          "BUG: executor-detected bug",
779				noStackTrace: true,
780			},
781			{
782				title: compile("BUG: memory leak"),
783				fmt:   "memory leak in %[1]v",
784				stack: &stackFmt{
785					parts: []*regexp.Regexp{
786						compile("backtrace:"),
787						parseStackTrace,
788					},
789					skip: []string{"kmemleak", "kmalloc", "kcalloc", "kzalloc",
790						"vmalloc", "kmem", "slab", "alloc", "create_object"},
791				},
792			},
793		},
794		[]*regexp.Regexp{
795			// CONFIG_DEBUG_OBJECTS output.
796			compile("ODEBUG:"),
797			// Android prints this sometimes during boot.
798			compile("Boot_DEBUG:"),
799			// pkg/host output in debug mode.
800			compile("BUG: no syscalls can create resource"),
801		},
802	},
803	{
804		[]byte("WARNING:"),
805		[]oopsFormat{
806			{
807				title: compile("WARNING: .*lib/debugobjects\\.c.* debug_print_object"),
808				fmt:   "WARNING: ODEBUG bug in %[1]v",
809				// Skip all users of ODEBUG as well.
810				stack: warningStackFmt("debug_", "rcu", "hrtimer_", "timer_",
811					"work_", "percpu_", "kmem_", "slab_", "kfree", "vunmap", "vfree"),
812			},
813			{
814				title: compile("WARNING: .*mm/usercopy\\.c.* usercopy_warn"),
815				fmt:   "WARNING: bad usercopy in %[1]v",
816				stack: warningStackFmt("usercopy", "__check"),
817			},
818			{
819				title: compile("WARNING: .*lib/kobject\\.c.* kobject_"),
820				fmt:   "WARNING: kobject bug in %[1]v",
821				stack: warningStackFmt("kobject_"),
822			},
823			{
824				title: compile("WARNING: .*fs/proc/generic\\.c.* proc_register"),
825				fmt:   "WARNING: proc registration bug in %[1]v",
826				stack: warningStackFmt("proc_"),
827			},
828			{
829				title: compile("WARNING: .*lib/refcount\\.c.* refcount_"),
830				fmt:   "WARNING: refcount bug in %[1]v",
831				stack: warningStackFmt("refcount"),
832			},
833			{
834				title: compile("WARNING: .*kernel/locking/lockdep\\.c.*lock_"),
835				fmt:   "WARNING: locking bug in %[1]v",
836				stack: warningStackFmt(),
837			},
838			{
839				title:        compile("WARNING: lock held when returning to user space"),
840				report:       compile("WARNING: lock held when returning to user space(?:.*\\n)+?.*leaving the kernel with locks still held(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
841				fmt:          "WARNING: lock held when returning to user space in %[1]v",
842				noStackTrace: true,
843			},
844			{
845				title: compile("WARNING: .*mm/.*\\.c.* k?.?malloc"),
846				fmt:   "WARNING: kmalloc bug in %[1]v",
847				stack: warningStackFmt("kmalloc", "kcalloc", "kzalloc", "krealloc",
848					"vmalloc", "slab", "kmem"),
849			},
850			{
851				title: compile("WARNING: .* at {{SRC}} {{FUNC}}"),
852				fmt:   "WARNING in %[2]v",
853			},
854			{
855				title:  compile("WARNING: possible circular locking dependency detected"),
856				report: compile("WARNING: possible circular locking dependency detected(?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
857				fmt:    "possible deadlock in %[1]v",
858			},
859			{
860				title:  compile("WARNING: possible irq lock inversion dependency detected"),
861				report: compile("WARNING: possible irq lock inversion dependency detected(?:.*\\n)+?.*just changed the state of lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
862				fmt:    "possible deadlock in %[1]v",
863			},
864			{
865				title:  compile("WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detecte"),
866				report: compile("WARNING: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected(?:.*\\n)+?.*is trying to acquire(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
867				fmt:    "possible deadlock in %[1]v",
868			},
869			{
870				title:  compile("WARNING: possible recursive locking detected"),
871				report: compile("WARNING: possible recursive locking detected(?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
872				fmt:    "possible deadlock in %[1]v",
873			},
874			{
875				title:  compile("WARNING: inconsistent lock state"),
876				report: compile("WARNING: inconsistent lock state(?:.*\\n)+?.*takes(?:.*\\n)+?.*at: (?:{{PC}} +)?{{FUNC}}"),
877				fmt:    "inconsistent lock state in %[1]v",
878			},
879			{
880				title:  compile("WARNING: suspicious RCU usage"),
881				report: compile("WARNING: suspicious RCU usage(?:.*\n)+?.*?{{SRC}}"),
882				fmt:    "WARNING: suspicious RCU usage in %[2]v",
883				stack: &stackFmt{
884					parts: []*regexp.Regexp{
885						compile("Call Trace:"),
886						parseStackTrace,
887					},
888					skip: []string{"rcu", "kmem", "slab", "kmalloc",
889						"vmalloc", "kcalloc", "kzalloc"},
890				},
891			},
892			{
893				title:        compile("WARNING: kernel stack regs at [0-9a-f]+ in [^ ]* has bad '([^']+)' value"),
894				fmt:          "WARNING: kernel stack regs has bad '%[1]v' value",
895				noStackTrace: true,
896			},
897			{
898				title:        compile("WARNING: kernel stack frame pointer at [0-9a-f]+ in [^ ]* has bad value"),
899				fmt:          "WARNING: kernel stack frame pointer has bad value",
900				noStackTrace: true,
901			},
902			{
903				title:  compile("WARNING: bad unlock balance detected!"),
904				report: compile("WARNING: bad unlock balance detected!(?:.*\\n){0,15}?.*is trying to release lock(?:.*\\n){0,15}?.*{{PC}} +{{FUNC}}"),
905				fmt:    "WARNING: bad unlock balance in %[1]v",
906			},
907			{
908				title:  compile("WARNING: held lock freed!"),
909				report: compile("WARNING: held lock freed!(?:.*\\n)+?.*{{PC}} +{{FUNC}}"),
910				fmt:    "WARNING: held lock freed in %[1]v",
911			},
912			{
913				title:        compile("WARNING: kernel stack regs .* has bad 'bp' value"),
914				fmt:          "WARNING: kernel stack regs has bad value",
915				noStackTrace: true,
916			},
917			{
918				title:        compile("WARNING: kernel stack frame pointer .* has bad value"),
919				fmt:          "WARNING: kernel stack regs has bad value",
920				noStackTrace: true,
921			},
922		},
923		[]*regexp.Regexp{
924			compile("WARNING: /etc/ssh/moduli does not exist, using fixed modulus"), // printed by sshd
925		},
926	},
927	{
928		[]byte("INFO:"),
929		[]oopsFormat{
930			{
931				title:  compile("INFO: possible circular locking dependency detected"),
932				report: compile("INFO: possible circular locking dependency detected \\](?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
933				fmt:    "possible deadlock in %[1]v",
934			},
935			{
936				title:  compile("INFO: possible irq lock inversion dependency detected"),
937				report: compile("INFO: possible irq lock inversion dependency detected \\](?:.*\\n)+?.*just changed the state of lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
938				fmt:    "possible deadlock in %[1]v",
939			},
940			{
941				title:  compile("INFO: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected"),
942				report: compile("INFO: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected \\](?:.*\\n)+?.*is trying to acquire(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
943				fmt:    "possible deadlock in %[1]v",
944			},
945			{
946				title:  compile("INFO: possible recursive locking detected"),
947				report: compile("INFO: possible recursive locking detected \\](?:.*\\n)+?.*is trying to acquire lock(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
948				fmt:    "possible deadlock in %[1]v",
949			},
950			{
951				title:  compile("INFO: inconsistent lock state"),
952				report: compile("INFO: inconsistent lock state \\](?:.*\\n)+?.*takes(?:.*\\n)+?.*at: {{PC}} +{{FUNC}}"),
953				fmt:    "inconsistent lock state in %[1]v",
954			},
955			{
956				title: linuxRcuStall,
957				fmt:   "INFO: rcu detected stall in %[1]v",
958				stack: &stackFmt{
959					parts: []*regexp.Regexp{
960						compile("apic_timer_interrupt"),
961						parseStackTrace,
962					},
963					skip: []string{"apic_timer_interrupt", "rcu"},
964				},
965			},
966			{
967				title: linuxRcuStall,
968				fmt:   "INFO: rcu detected stall in %[1]v",
969				stack: &stackFmt{
970					parts: []*regexp.Regexp{
971						linuxRipFrame,
972						parseStackTrace,
973					},
974					skip: []string{"apic_timer_interrupt", "rcu"},
975				},
976			},
977			{
978				title: compile("INFO: trying to register non-static key"),
979				fmt:   "INFO: trying to register non-static key in %[1]v",
980				stack: &stackFmt{
981					parts: []*regexp.Regexp{
982						compile("Call Trace:"),
983						parseStackTrace,
984					},
985					skip: []string{"stack", "lock", "IRQ"},
986				},
987			},
988			{
989				title:  compile("INFO: suspicious RCU usage"),
990				report: compile("INFO: suspicious RCU usage(?:.*\n)+?.*?{{SRC}}"),
991				fmt:    "INFO: suspicious RCU usage in %[2]v",
992				stack: &stackFmt{
993					parts: []*regexp.Regexp{
994						compile("Call Trace:"),
995						parseStackTrace,
996					},
997					skip: []string{"rcu", "kmem", "slab", "kmalloc",
998						"vmalloc", "kcalloc", "kzalloc"},
999				},
1000			},
1001			{
1002				title: compile("INFO: task .* blocked for more than [0-9]+ seconds"),
1003				fmt:   "INFO: task hung in %[1]v",
1004				stack: &stackFmt{
1005					parts: []*regexp.Regexp{
1006						compile("Call Trace:"),
1007						parseStackTrace,
1008					},
1009					skip: []string{"sched", "_lock", "down", "completion", "kthread",
1010						"wait", "synchronize"},
1011				},
1012			},
1013			{
1014				// This gets captured for corrupted old-style KASAN reports.
1015				title:     compile("INFO: (Freed|Allocated) in (.*)"),
1016				fmt:       "INFO: %[1]v in %[2]v",
1017				corrupted: true,
1018			},
1019		},
1020		[]*regexp.Regexp{
1021			compile("INFO: lockdep is turned off"),
1022			compile("INFO: Stall ended before state dump start"),
1023			compile("INFO: NMI handler"),
1024			compile("(handler|interrupt).*took too long"),
1025			compile("_INFO::"),                                       // Android can print this during boot.
1026			compile("INFO: sys_.* is not present in /proc/kallsyms"), // pkg/host output in debug mode
1027			compile("INFO: no syscalls can create resource"),         // pkg/host output in debug mode
1028		},
1029	},
1030	{
1031		[]byte("Unable to handle kernel paging request"),
1032		[]oopsFormat{
1033			{
1034				title:  compile("Unable to handle kernel paging request"),
1035				report: compile("Unable to handle kernel paging request(?:.*\\n)+?.*PC is at {{FUNC}}"),
1036				fmt:    "unable to handle kernel paging request in %[1]v",
1037			},
1038		},
1039		[]*regexp.Regexp{},
1040	},
1041	{
1042		[]byte("general protection fault:"),
1043		[]oopsFormat{
1044			{
1045				title: compile("general protection fault:"),
1046				fmt:   "general protection fault in %[1]v",
1047				stack: &stackFmt{
1048					parts: []*regexp.Regexp{
1049						linuxRipFrame,
1050						compile("Call Trace:"),
1051						parseStackTrace,
1052					},
1053				},
1054			},
1055		},
1056		[]*regexp.Regexp{},
1057	},
1058	{
1059		[]byte("Kernel panic"),
1060		[]oopsFormat{
1061			{
1062				title: compile("Kernel panic - not syncing: Attempted to kill init!"),
1063				fmt:   "kernel panic: Attempted to kill init!",
1064			},
1065			{
1066				title: compile("Kernel panic - not syncing: Couldn't open N_TTY ldisc for [^ ]+ --- error -[0-9]+"),
1067				fmt:   "kernel panic: Couldn't open N_TTY ldisc",
1068			},
1069			{
1070				// 'kernel panic: Fatal exception' is usually printed after BUG,
1071				// so if we captured it as a report description, that means the
1072				// report got truncated and we missed the actual BUG header.
1073				title:     compile("Kernel panic - not syncing: Fatal exception"),
1074				fmt:       "kernel panic: Fatal exception",
1075				corrupted: true,
1076			},
1077			{
1078				// Same, but for WARNINGs and KASAN reports.
1079				title:     compile("Kernel panic - not syncing: panic_on_warn set"),
1080				fmt:       "kernel panic: panic_on_warn set",
1081				corrupted: true,
1082			},
1083			{
1084				// Same, but for task hung reports.
1085				title:     compile("Kernel panic - not syncing: hung_task: blocked tasks"),
1086				fmt:       "kernel panic: hung_task: blocked tasks",
1087				corrupted: true,
1088			},
1089			{
1090				title: compile("Kernel panic - not syncing: (.*)"),
1091				fmt:   "kernel panic: %[1]v",
1092			},
1093		},
1094		[]*regexp.Regexp{},
1095	},
1096	{
1097		[]byte("kernel BUG"),
1098		[]oopsFormat{
1099			{
1100				title: compile("kernel BUG at mm/usercopy.c"),
1101				fmt:   "BUG: bad usercopy in %[1]v",
1102				stack: &stackFmt{
1103					parts: []*regexp.Regexp{
1104						compile("Call Trace:"),
1105						parseStackTrace,
1106					},
1107				},
1108			},
1109			{
1110				title: compile("kernel BUG at lib/list_debug.c"),
1111				fmt:   "BUG: corrupted list in %[1]v",
1112				stack: &stackFmt{
1113					parts: []*regexp.Regexp{
1114						compile("Call Trace:"),
1115						parseStackTrace,
1116					},
1117				},
1118			},
1119		},
1120		[]*regexp.Regexp{},
1121	},
1122	{
1123		[]byte("Kernel BUG"),
1124		[]oopsFormat{
1125			{
1126				title: compile("Kernel BUG (.*)"),
1127				fmt:   "kernel BUG %[1]v",
1128			},
1129		},
1130		[]*regexp.Regexp{},
1131	},
1132	{
1133		[]byte("BUG kmalloc-"),
1134		[]oopsFormat{
1135			{
1136				title: compile("BUG kmalloc-.*: Object already free"),
1137				fmt:   "BUG: Object already free",
1138			},
1139		},
1140		[]*regexp.Regexp{},
1141	},
1142	{
1143		[]byte("divide error:"),
1144		[]oopsFormat{
1145			{
1146				title:  compile("divide error: "),
1147				report: compile("divide error: (?:.*\\n)+?.*RIP: [0-9]+:(?:{{PC}} +{{PC}} +)?{{FUNC}}"),
1148				fmt:    "divide error in %[1]v",
1149			},
1150		},
1151		[]*regexp.Regexp{},
1152	},
1153	{
1154		[]byte("invalid opcode:"),
1155		[]oopsFormat{
1156			{
1157				title:  compile("invalid opcode: "),
1158				report: compile("invalid opcode: (?:.*\\n)+?.*RIP: [0-9]+:{{PC}} +{{PC}} +{{FUNC}}"),
1159				fmt:    "invalid opcode in %[1]v",
1160			},
1161		},
1162		[]*regexp.Regexp{},
1163	},
1164	{
1165		[]byte("UBSAN:"),
1166		[]oopsFormat{
1167			{
1168				title: compile("UBSAN: (.*)"),
1169				fmt:   "UBSAN: %[1]v",
1170			},
1171		},
1172		[]*regexp.Regexp{},
1173	},
1174	{
1175		[]byte("Booting the kernel."),
1176		[]oopsFormat{
1177			{
1178				title:        compile("Booting the kernel."),
1179				fmt:          "unexpected kernel reboot",
1180				noStackTrace: true,
1181			},
1182		},
1183		[]*regexp.Regexp{},
1184	},
1185	{
1186		[]byte("unregister_netdevice: waiting for"),
1187		[]oopsFormat{
1188			{
1189				title:        compile("unregister_netdevice: waiting for (?:.*) to become free"),
1190				fmt:          "unregister_netdevice: waiting for DEV to become free",
1191				noStackTrace: true,
1192			},
1193		},
1194		[]*regexp.Regexp{},
1195	},
1196}
1197