1// doc generates HTML files from the comments in header files.
2//
3// doc expects to be given the path to a JSON file via the --config option.
4// From that JSON (which is defined by the Config struct) it reads a list of
5// header file locations and generates HTML files for each in the current
6// directory.
7
8package main
9
10import (
11	"bufio"
12	"encoding/json"
13	"errors"
14	"flag"
15	"fmt"
16	"html/template"
17	"io/ioutil"
18	"os"
19	"path/filepath"
20	"strings"
21)
22
23// Config describes the structure of the config JSON file.
24type Config struct {
25	// BaseDirectory is a path to which other paths in the file are
26	// relative.
27	BaseDirectory string
28	Sections      []ConfigSection
29}
30
31type ConfigSection struct {
32	Name string
33	// Headers is a list of paths to header files.
34	Headers []string
35}
36
37// HeaderFile is the internal representation of a header file.
38type HeaderFile struct {
39	// Name is the basename of the header file (e.g. "ex_data.html").
40	Name string
41	// Preamble contains a comment for the file as a whole. Each string
42	// is a separate paragraph.
43	Preamble []string
44	Sections []HeaderSection
45	// AllDecls maps all decls to their URL fragments.
46	AllDecls map[string]string
47}
48
49type HeaderSection struct {
50	// Preamble contains a comment for a group of functions.
51	Preamble []string
52	Decls    []HeaderDecl
53	// Anchor, if non-empty, is the URL fragment to use in anchor tags.
54	Anchor string
55	// IsPrivate is true if the section contains private functions (as
56	// indicated by its name).
57	IsPrivate bool
58}
59
60type HeaderDecl struct {
61	// Comment contains a comment for a specific function. Each string is a
62	// paragraph. Some paragraph may contain \n runes to indicate that they
63	// are preformatted.
64	Comment []string
65	// Name contains the name of the function, if it could be extracted.
66	Name string
67	// Decl contains the preformatted C declaration itself.
68	Decl string
69	// Anchor, if non-empty, is the URL fragment to use in anchor tags.
70	Anchor string
71}
72
73const (
74	cppGuard     = "#if defined(__cplusplus)"
75	commentStart = "/* "
76	commentEnd   = " */"
77)
78
79func extractComment(lines []string, lineNo int) (comment []string, rest []string, restLineNo int, err error) {
80	if len(lines) == 0 {
81		return nil, lines, lineNo, nil
82	}
83
84	restLineNo = lineNo
85	rest = lines
86
87	if !strings.HasPrefix(rest[0], commentStart) {
88		panic("extractComment called on non-comment")
89	}
90	commentParagraph := rest[0][len(commentStart):]
91	rest = rest[1:]
92	restLineNo++
93
94	for len(rest) > 0 {
95		i := strings.Index(commentParagraph, commentEnd)
96		if i >= 0 {
97			if i != len(commentParagraph)-len(commentEnd) {
98				err = fmt.Errorf("garbage after comment end on line %d", restLineNo)
99				return
100			}
101			commentParagraph = commentParagraph[:i]
102			if len(commentParagraph) > 0 {
103				comment = append(comment, commentParagraph)
104			}
105			return
106		}
107
108		line := rest[0]
109		if !strings.HasPrefix(line, " *") {
110			err = fmt.Errorf("comment doesn't start with block prefix on line %d: %s", restLineNo, line)
111			return
112		}
113		if len(line) == 2 || line[2] != '/' {
114			line = line[2:]
115		}
116		if strings.HasPrefix(line, "   ") {
117			/* Identing the lines of a paragraph marks them as
118			* preformatted. */
119			if len(commentParagraph) > 0 {
120				commentParagraph += "\n"
121			}
122			line = line[3:]
123		}
124		if len(line) > 0 {
125			commentParagraph = commentParagraph + line
126			if len(commentParagraph) > 0 && commentParagraph[0] == ' ' {
127				commentParagraph = commentParagraph[1:]
128			}
129		} else {
130			comment = append(comment, commentParagraph)
131			commentParagraph = ""
132		}
133		rest = rest[1:]
134		restLineNo++
135	}
136
137	err = errors.New("hit EOF in comment")
138	return
139}
140
141func extractDecl(lines []string, lineNo int) (decl string, rest []string, restLineNo int, err error) {
142	if len(lines) == 0 {
143		return "", lines, lineNo, nil
144	}
145
146	rest = lines
147	restLineNo = lineNo
148
149	var stack []rune
150	for len(rest) > 0 {
151		line := rest[0]
152		for _, c := range line {
153			switch c {
154			case '(', '{', '[':
155				stack = append(stack, c)
156			case ')', '}', ']':
157				if len(stack) == 0 {
158					err = fmt.Errorf("unexpected %c on line %d", c, restLineNo)
159					return
160				}
161				var expected rune
162				switch c {
163				case ')':
164					expected = '('
165				case '}':
166					expected = '{'
167				case ']':
168					expected = '['
169				default:
170					panic("internal error")
171				}
172				if last := stack[len(stack)-1]; last != expected {
173					err = fmt.Errorf("found %c when expecting %c on line %d", c, last, restLineNo)
174					return
175				}
176				stack = stack[:len(stack)-1]
177			}
178		}
179		if len(decl) > 0 {
180			decl += "\n"
181		}
182		decl += line
183		rest = rest[1:]
184		restLineNo++
185
186		if len(stack) == 0 && (len(decl) == 0 || decl[len(decl)-1] != '\\') {
187			break
188		}
189	}
190
191	return
192}
193
194func skipLine(s string) string {
195	i := strings.Index(s, "\n")
196	if i > 0 {
197		return s[i:]
198	}
199	return ""
200}
201
202func getNameFromDecl(decl string) (string, bool) {
203	for strings.HasPrefix(decl, "#if") || strings.HasPrefix(decl, "#elif") {
204		decl = skipLine(decl)
205	}
206	if strings.HasPrefix(decl, "struct ") {
207		return "", false
208	}
209	if strings.HasPrefix(decl, "#define ") {
210		// This is a preprocessor #define. The name is the next symbol.
211		decl = strings.TrimPrefix(decl, "#define ")
212		for len(decl) > 0 && decl[0] == ' ' {
213			decl = decl[1:]
214		}
215		i := strings.IndexAny(decl, "( ")
216		if i < 0 {
217			return "", false
218		}
219		return decl[:i], true
220	}
221	decl = strings.TrimPrefix(decl, "OPENSSL_EXPORT ")
222	decl = strings.TrimPrefix(decl, "STACK_OF(")
223	decl = strings.TrimPrefix(decl, "LHASH_OF(")
224	i := strings.Index(decl, "(")
225	if i < 0 {
226		return "", false
227	}
228	j := strings.LastIndex(decl[:i], " ")
229	if j < 0 {
230		return "", false
231	}
232	for j+1 < len(decl) && decl[j+1] == '*' {
233		j++
234	}
235	return decl[j+1 : i], true
236}
237
238func sanitizeAnchor(name string) string {
239	return strings.Replace(name, " ", "-", -1)
240}
241
242func isPrivateSection(name string) bool {
243	return strings.HasPrefix(name, "Private functions") || strings.HasPrefix(name, "Private structures") || strings.Contains(name, "(hidden)")
244}
245
246func (config *Config) parseHeader(path string) (*HeaderFile, error) {
247	headerPath := filepath.Join(config.BaseDirectory, path)
248
249	headerFile, err := os.Open(headerPath)
250	if err != nil {
251		return nil, err
252	}
253	defer headerFile.Close()
254
255	scanner := bufio.NewScanner(headerFile)
256	var lines, oldLines []string
257	for scanner.Scan() {
258		lines = append(lines, scanner.Text())
259	}
260	if err := scanner.Err(); err != nil {
261		return nil, err
262	}
263
264	lineNo := 0
265	found := false
266	for i, line := range lines {
267		lineNo++
268		if line == cppGuard {
269			lines = lines[i+1:]
270			lineNo++
271			found = true
272			break
273		}
274	}
275
276	if !found {
277		return nil, errors.New("no C++ guard found")
278	}
279
280	if len(lines) == 0 || lines[0] != "extern \"C\" {" {
281		return nil, errors.New("no extern \"C\" found after C++ guard")
282	}
283	lineNo += 2
284	lines = lines[2:]
285
286	header := &HeaderFile{
287		Name:     filepath.Base(path),
288		AllDecls: make(map[string]string),
289	}
290
291	for i, line := range lines {
292		lineNo++
293		if len(line) > 0 {
294			lines = lines[i:]
295			break
296		}
297	}
298
299	oldLines = lines
300	if len(lines) > 0 && strings.HasPrefix(lines[0], commentStart) {
301		comment, rest, restLineNo, err := extractComment(lines, lineNo)
302		if err != nil {
303			return nil, err
304		}
305
306		if len(rest) > 0 && len(rest[0]) == 0 {
307			if len(rest) < 2 || len(rest[1]) != 0 {
308				return nil, errors.New("preamble comment should be followed by two blank lines")
309			}
310			header.Preamble = comment
311			lineNo = restLineNo + 2
312			lines = rest[2:]
313		} else {
314			lines = oldLines
315		}
316	}
317
318	allAnchors := make(map[string]struct{})
319
320	for {
321		// Start of a section.
322		if len(lines) == 0 {
323			return nil, errors.New("unexpected end of file")
324		}
325		line := lines[0]
326		if line == cppGuard {
327			break
328		}
329
330		if len(line) == 0 {
331			return nil, fmt.Errorf("blank line at start of section on line %d", lineNo)
332		}
333
334		var section HeaderSection
335
336		if strings.HasPrefix(line, commentStart) {
337			comment, rest, restLineNo, err := extractComment(lines, lineNo)
338			if err != nil {
339				return nil, err
340			}
341			if len(rest) > 0 && len(rest[0]) == 0 {
342				anchor := sanitizeAnchor(firstSentence(comment))
343				if len(anchor) > 0 {
344					if _, ok := allAnchors[anchor]; ok {
345						return nil, fmt.Errorf("duplicate anchor: %s", anchor)
346					}
347					allAnchors[anchor] = struct{}{}
348				}
349
350				section.Preamble = comment
351				section.IsPrivate = len(comment) > 0 && isPrivateSection(comment[0])
352				section.Anchor = anchor
353				lines = rest[1:]
354				lineNo = restLineNo + 1
355			}
356		}
357
358		for len(lines) > 0 {
359			line := lines[0]
360			if len(line) == 0 {
361				lines = lines[1:]
362				lineNo++
363				break
364			}
365			if line == cppGuard {
366				return nil, errors.New("hit ending C++ guard while in section")
367			}
368
369			var comment []string
370			var decl string
371			if strings.HasPrefix(line, commentStart) {
372				comment, lines, lineNo, err = extractComment(lines, lineNo)
373				if err != nil {
374					return nil, err
375				}
376			}
377			if len(lines) == 0 {
378				return nil, errors.New("expected decl at EOF")
379			}
380			decl, lines, lineNo, err = extractDecl(lines, lineNo)
381			if err != nil {
382				return nil, err
383			}
384			name, ok := getNameFromDecl(decl)
385			if !ok {
386				name = ""
387			}
388			if last := len(section.Decls) - 1; len(name) == 0 && len(comment) == 0 && last >= 0 {
389				section.Decls[last].Decl += "\n" + decl
390			} else {
391				// As a matter of style, comments should start
392				// with the name of the thing that they are
393				// commenting on. We make an exception here for
394				// #defines (because we often have blocks of
395				// them) and collective comments, which are
396				// detected by starting with “The” or “These”.
397				if len(comment) > 0 &&
398					!strings.HasPrefix(comment[0], name) &&
399					!strings.HasPrefix(decl, "#define ") &&
400					!strings.HasPrefix(comment[0], "The ") &&
401					!strings.HasPrefix(comment[0], "These ") {
402					return nil, fmt.Errorf("Comment for %q doesn't seem to match just above %s:%d\n", name, path, lineNo)
403				}
404				anchor := sanitizeAnchor(name)
405				// TODO(davidben): Enforce uniqueness. This is
406				// skipped because #ifdefs currently result in
407				// duplicate table-of-contents entries.
408				allAnchors[anchor] = struct{}{}
409
410				header.AllDecls[name] = anchor
411
412				section.Decls = append(section.Decls, HeaderDecl{
413					Comment: comment,
414					Name:    name,
415					Decl:    decl,
416					Anchor:  anchor,
417				})
418			}
419
420			if len(lines) > 0 && len(lines[0]) == 0 {
421				lines = lines[1:]
422				lineNo++
423			}
424		}
425
426		header.Sections = append(header.Sections, section)
427	}
428
429	return header, nil
430}
431
432func firstSentence(paragraphs []string) string {
433	if len(paragraphs) == 0 {
434		return ""
435	}
436	s := paragraphs[0]
437	i := strings.Index(s, ". ")
438	if i >= 0 {
439		return s[:i]
440	}
441	if lastIndex := len(s) - 1; s[lastIndex] == '.' {
442		return s[:lastIndex]
443	}
444	return s
445}
446
447func markupPipeWords(allDecls map[string]string, s string) template.HTML {
448	ret := ""
449
450	for {
451		i := strings.Index(s, "|")
452		if i == -1 {
453			ret += s
454			break
455		}
456		ret += s[:i]
457		s = s[i+1:]
458
459		i = strings.Index(s, "|")
460		j := strings.Index(s, " ")
461		if i > 0 && (j == -1 || j > i) {
462			ret += "<tt>"
463			anchor, isLink := allDecls[s[:i]]
464			if isLink {
465				ret += fmt.Sprintf("<a href=\"%s\">", template.HTMLEscapeString(anchor))
466			}
467			ret += s[:i]
468			if isLink {
469				ret += "</a>"
470			}
471			ret += "</tt>"
472			s = s[i+1:]
473		} else {
474			ret += "|"
475		}
476	}
477
478	return template.HTML(ret)
479}
480
481func markupFirstWord(s template.HTML) template.HTML {
482	start := 0
483again:
484	end := strings.Index(string(s[start:]), " ")
485	if end > 0 {
486		end += start
487		w := strings.ToLower(string(s[start:end]))
488		// The first word was already marked up as an HTML tag. Don't
489		// mark it up further.
490		if strings.ContainsRune(w, '<') {
491			return s
492		}
493		if w == "a" || w == "an" {
494			start = end + 1
495			goto again
496		}
497		return s[:start] + "<span class=\"first-word\">" + s[start:end] + "</span>" + s[end:]
498	}
499	return s
500}
501
502func newlinesToBR(html template.HTML) template.HTML {
503	s := string(html)
504	if !strings.Contains(s, "\n") {
505		return html
506	}
507	s = strings.Replace(s, "\n", "<br>", -1)
508	s = strings.Replace(s, " ", "&nbsp;", -1)
509	return template.HTML(s)
510}
511
512func generate(outPath string, config *Config) (map[string]string, error) {
513	allDecls := make(map[string]string)
514
515	headerTmpl := template.New("headerTmpl")
516	headerTmpl.Funcs(template.FuncMap{
517		"firstSentence":   firstSentence,
518		"markupPipeWords": func(s string) template.HTML { return markupPipeWords(allDecls, s) },
519		"markupFirstWord": markupFirstWord,
520		"newlinesToBR":    newlinesToBR,
521	})
522	headerTmpl, err := headerTmpl.Parse(`<!DOCTYPE html>
523<html>
524  <head>
525    <title>BoringSSL - {{.Name}}</title>
526    <meta charset="utf-8">
527    <link rel="stylesheet" type="text/css" href="doc.css">
528  </head>
529
530  <body>
531    <div id="main">
532    <h2>{{.Name}}</h2>
533
534    {{range .Preamble}}<p>{{. | html | markupPipeWords}}</p>{{end}}
535
536    <ol>
537      {{range .Sections}}
538        {{if not .IsPrivate}}
539          {{if .Anchor}}<li class="header"><a href="#{{.Anchor}}">{{.Preamble | firstSentence | html | markupPipeWords}}</a></li>{{end}}
540          {{range .Decls}}
541            {{if .Anchor}}<li><a href="#{{.Anchor}}"><tt>{{.Name}}</tt></a></li>{{end}}
542          {{end}}
543        {{end}}
544      {{end}}
545    </ol>
546
547    {{range .Sections}}
548      {{if not .IsPrivate}}
549        <div class="section" {{if .Anchor}}id="{{.Anchor}}"{{end}}>
550        {{if .Preamble}}
551          <div class="sectionpreamble">
552          {{range .Preamble}}<p>{{. | html | markupPipeWords}}</p>{{end}}
553          </div>
554        {{end}}
555
556        {{range .Decls}}
557          <div class="decl" {{if .Anchor}}id="{{.Anchor}}"{{end}}>
558          {{range .Comment}}
559            <p>{{. | html | markupPipeWords | newlinesToBR | markupFirstWord}}</p>
560          {{end}}
561          <pre>{{.Decl}}</pre>
562          </div>
563        {{end}}
564        </div>
565      {{end}}
566    {{end}}
567    </div>
568  </body>
569</html>`)
570	if err != nil {
571		return nil, err
572	}
573
574	headerDescriptions := make(map[string]string)
575	var headers []*HeaderFile
576
577	for _, section := range config.Sections {
578		for _, headerPath := range section.Headers {
579			header, err := config.parseHeader(headerPath)
580			if err != nil {
581				return nil, errors.New("while parsing " + headerPath + ": " + err.Error())
582			}
583			headerDescriptions[header.Name] = firstSentence(header.Preamble)
584			headers = append(headers, header)
585
586			for name, anchor := range header.AllDecls {
587				allDecls[name] = fmt.Sprintf("%s#%s", header.Name+".html", anchor)
588			}
589		}
590	}
591
592	for _, header := range headers {
593		filename := filepath.Join(outPath, header.Name+".html")
594		file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
595		if err != nil {
596			panic(err)
597		}
598		defer file.Close()
599		if err := headerTmpl.Execute(file, header); err != nil {
600			return nil, err
601		}
602	}
603
604	return headerDescriptions, nil
605}
606
607func generateIndex(outPath string, config *Config, headerDescriptions map[string]string) error {
608	indexTmpl := template.New("indexTmpl")
609	indexTmpl.Funcs(template.FuncMap{
610		"baseName": filepath.Base,
611		"headerDescription": func(header string) string {
612			return headerDescriptions[header]
613		},
614	})
615	indexTmpl, err := indexTmpl.Parse(`<!DOCTYPE html5>
616
617  <head>
618    <title>BoringSSL - Headers</title>
619    <meta charset="utf-8">
620    <link rel="stylesheet" type="text/css" href="doc.css">
621  </head>
622
623  <body>
624    <div id="main">
625      <table>
626        {{range .Sections}}
627	  <tr class="header"><td colspan="2">{{.Name}}</td></tr>
628	  {{range .Headers}}
629	    <tr><td><a href="{{. | baseName}}.html">{{. | baseName}}</a></td><td>{{. | baseName | headerDescription}}</td></tr>
630	  {{end}}
631	{{end}}
632      </table>
633    </div>
634  </body>
635</html>`)
636
637	if err != nil {
638		return err
639	}
640
641	file, err := os.OpenFile(filepath.Join(outPath, "headers.html"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
642	if err != nil {
643		panic(err)
644	}
645	defer file.Close()
646
647	if err := indexTmpl.Execute(file, config); err != nil {
648		return err
649	}
650
651	return nil
652}
653
654func copyFile(outPath string, inFilePath string) error {
655	bytes, err := ioutil.ReadFile(inFilePath)
656	if err != nil {
657		return err
658	}
659	return ioutil.WriteFile(filepath.Join(outPath, filepath.Base(inFilePath)), bytes, 0666)
660}
661
662func main() {
663	var (
664		configFlag *string = flag.String("config", "doc.config", "Location of config file")
665		outputDir  *string = flag.String("out", ".", "Path to the directory where the output will be written")
666		config     Config
667	)
668
669	flag.Parse()
670
671	if len(*configFlag) == 0 {
672		fmt.Printf("No config file given by --config\n")
673		os.Exit(1)
674	}
675
676	if len(*outputDir) == 0 {
677		fmt.Printf("No output directory given by --out\n")
678		os.Exit(1)
679	}
680
681	configBytes, err := ioutil.ReadFile(*configFlag)
682	if err != nil {
683		fmt.Printf("Failed to open config file: %s\n", err)
684		os.Exit(1)
685	}
686
687	if err := json.Unmarshal(configBytes, &config); err != nil {
688		fmt.Printf("Failed to parse config file: %s\n", err)
689		os.Exit(1)
690	}
691
692	headerDescriptions, err := generate(*outputDir, &config)
693	if err != nil {
694		fmt.Printf("Failed to generate output: %s\n", err)
695		os.Exit(1)
696	}
697
698	if err := generateIndex(*outputDir, &config, headerDescriptions); err != nil {
699		fmt.Printf("Failed to generate index: %s\n", err)
700		os.Exit(1)
701	}
702
703	if err := copyFile(*outputDir, "doc.css"); err != nil {
704		fmt.Printf("Failed to copy static file: %s\n", err)
705		os.Exit(1)
706	}
707}
708