<lambda>null1 package org.jetbrains.dokka
2
3 import com.intellij.psi.*
4 import com.intellij.psi.impl.source.javadoc.PsiDocTagValueImpl
5 import com.intellij.psi.impl.source.tree.JavaDocElementType
6 import com.intellij.psi.javadoc.*
7 import com.intellij.psi.util.PsiTreeUtil
8 import com.intellij.util.IncorrectOperationException
9 import org.jetbrains.dokka.Model.CodeNode
10 import org.jetbrains.kotlin.utils.join
11 import org.jetbrains.kotlin.utils.keysToMap
12 import org.jsoup.Jsoup
13 import org.jsoup.nodes.Element
14 import org.jsoup.nodes.Node
15 import org.jsoup.nodes.TextNode
16 import java.io.File
17 import java.net.URI
18 import java.util.regex.Pattern
19
20 private val NAME_TEXT = Pattern.compile("(\\S+)(.*)", Pattern.DOTALL)
21 private val TEXT = Pattern.compile("(\\S+)\\s*(.*)", Pattern.DOTALL)
22
23 data class JavadocParseResult(
24 val content: Content,
25 val deprecatedContent: Content?,
26 val attributeRefs: List<String>,
27 val apiLevel: DocumentationNode? = null,
28 val deprecatedLevel: DocumentationNode? = null,
29 val artifactId: DocumentationNode? = null,
30 val attribute: DocumentationNode? = null
31 ) {
32 companion object {
33 val Empty = JavadocParseResult(Content.Empty,
34 null,
35 emptyList(),
36 null,
37 null,
38 null
39 )
40 }
41 }
42
43 interface JavaDocumentationParser {
parseDocumentationnull44 fun parseDocumentation(element: PsiNamedElement): JavadocParseResult
45 }
46
47 class JavadocParser(
48 private val refGraph: NodeReferenceGraph,
49 private val logger: DokkaLogger,
50 private val signatureProvider: ElementSignatureProvider,
51 private val externalDocumentationLinkResolver: ExternalDocumentationLinkResolver
52 ) : JavaDocumentationParser {
53
54 private fun ContentSection.appendTypeElement(
55 signature: String,
56 selector: (DocumentationNode) -> DocumentationNode?
57 ) {
58 append(LazyContentBlock {
59 val node = refGraph.lookupOrWarn(signature, logger)?.let(selector)
60 if (node != null) {
61 it.append(NodeRenderContent(node, LanguageService.RenderMode.SUMMARY))
62 it.symbol(":")
63 it.text(" ")
64 }
65 })
66 }
67
68 override fun parseDocumentation(element: PsiNamedElement): JavadocParseResult {
69 val docComment = (element as? PsiDocCommentOwner)?.docComment
70 if (docComment == null) return JavadocParseResult.Empty
71 val result = MutableContent()
72 var deprecatedContent: Content? = null
73 val firstParagraph = ContentParagraph()
74 firstParagraph.convertJavadocElements(
75 docComment.descriptionElements.dropWhile { it.text.trim().isEmpty() },
76 element
77 )
78 val paragraphs = firstParagraph.children.dropWhile { it !is ContentParagraph }
79 firstParagraph.children.removeAll(paragraphs)
80 if (!firstParagraph.isEmpty()) {
81 result.append(firstParagraph)
82 }
83 paragraphs.forEach {
84 result.append(it)
85 }
86
87 if (element is PsiMethod) {
88 val tagsByName = element.searchInheritedTags()
89 for ((tagName, tags) in tagsByName) {
90 for ((tag, context) in tags) {
91 val section = result.addSection(javadocSectionDisplayName(tagName), tag.getSubjectName())
92 val signature = signatureProvider.signature(element)
93 when (tagName) {
94 "param" -> {
95 section.appendTypeElement(signature) {
96 it.details
97 .find { node -> node.kind == NodeKind.Parameter && node.name == tag.getSubjectName() }
98 ?.detailOrNull(NodeKind.Type)
99 }
100 }
101 "return" -> {
102 section.appendTypeElement(signature) { it.detailOrNull(NodeKind.Type) }
103 }
104 }
105 section.convertJavadocElements(tag.contentElements(), context)
106 }
107 }
108 }
109
110 val attrRefSignatures = mutableListOf<String>()
111 var since: DocumentationNode? = null
112 var deprecated: DocumentationNode? = null
113 var artifactId: DocumentationNode? = null
114 var attrName: String? = null
115 var attrDesc: Content? = null
116 var attr: DocumentationNode? = null
117 docComment.tags.forEach { tag ->
118 when (tag.name.toLowerCase()) {
119 "see" -> result.convertSeeTag(tag)
120 "deprecated" -> {
121 deprecatedContent = Content().apply {
122 convertJavadocElements(tag.contentElements(), element)
123 }
124 }
125 "attr" -> {
126 when (tag.valueElement?.text) {
127 "ref" ->
128 tag.getAttrRef(element)?.let {
129 attrRefSignatures.add(it)
130 }
131 "name" -> attrName = tag.getAttrName()
132 "description" -> attrDesc = tag.getAttrDesc(element)
133 }
134 }
135 "since", "apisince" -> {
136 since = DocumentationNode(tag.getApiLevel() ?: "", Content.Empty, NodeKind.ApiLevel)
137 }
138 "deprecatedsince" -> {
139 deprecated = DocumentationNode(tag.getApiLevel() ?: "", Content.Empty, NodeKind.DeprecatedLevel)
140 }
141 "artifactid" -> {
142 artifactId = DocumentationNode(tag.artifactId() ?: "", Content.Empty, NodeKind.ArtifactId)
143 }
144 in tagsToInherit -> {
145 }
146 else -> {
147 val subjectName = tag.getSubjectName()
148 val section = result.addSection(javadocSectionDisplayName(tag.name), subjectName)
149 section.convertJavadocElements(tag.contentElements(), element)
150 }
151 }
152 }
153 attrName?.let { name ->
154 attr = DocumentationNode(name, attrDesc ?: Content.Empty, NodeKind.AttributeRef)
155 }
156 return JavadocParseResult(result, deprecatedContent, attrRefSignatures, since, deprecated, artifactId, attr)
157 }
158
159 private val tagsToInherit = setOf("param", "return", "throws")
160
161 private data class TagWithContext(val tag: PsiDocTag, val context: PsiNamedElement)
162
163 fun PsiDocTag.artifactId(): String? {
164 var artifactName: String? = null
165 if (dataElements.isNotEmpty()) {
166 artifactName = join(dataElements.map { it.text }, "")
167 }
168 return artifactName
169 }
170
171 fun PsiDocTag.getApiLevel(): String? {
172 if (dataElements.isNotEmpty()) {
173 val data = dataElements
174 if (data[0] is PsiDocTagValueImpl) {
175 val docTagValue = data[0]
176 if (docTagValue.firstChild != null) {
177 val apiLevel = docTagValue.firstChild
178 return apiLevel.text
179 }
180 }
181 }
182 return null
183 }
184
185 private fun PsiDocTag.getAttrRef(element: PsiNamedElement): String? {
186 if (dataElements.size > 1) {
187 val elementText = dataElements[1].text
188 try {
189 val linkComment = JavaPsiFacade.getInstance(project).elementFactory
190 .createDocCommentFromText("/** {@link $elementText} */", element)
191 val linkElement = PsiTreeUtil.getChildOfType(linkComment, PsiInlineDocTag::class.java)?.linkElement()
192 val signature = resolveInternalLink(linkElement)
193 val attrSignature = "AttrMain:$signature"
194 return attrSignature
195 } catch (e: IncorrectOperationException) {
196 return null
197 }
198 } else return null
199 }
200
201 private fun PsiDocTag.getAttrName(): String? {
202 if (dataElements.size > 1) {
203 val nameMatcher = NAME_TEXT.matcher(dataElements[1].text)
204 if (nameMatcher.matches()) {
205 return nameMatcher.group(1)
206 } else {
207 return null
208 }
209 } else return null
210 }
211
212 private fun PsiDocTag.getAttrDesc(element: PsiNamedElement): Content? {
213 return Content().apply {
214 convertJavadocElementsToAttrDesc(contentElements(), element)
215 }
216 }
217
218 private fun PsiMethod.searchInheritedTags(): Map<String, Collection<TagWithContext>> {
219
220 val output = tagsToInherit.keysToMap { mutableMapOf<String?, TagWithContext>() }
221
222 fun recursiveSearch(methods: Array<PsiMethod>) {
223 for (method in methods) {
224 recursiveSearch(method.findSuperMethods())
225 }
226 for (method in methods) {
227 for (tag in method.docComment?.tags.orEmpty()) {
228 if (tag.name in tagsToInherit) {
229 output[tag.name]!![tag.getSubjectName()] = TagWithContext(tag, method)
230 }
231 }
232 }
233 }
234
235 recursiveSearch(arrayOf(this))
236 return output.mapValues { it.value.values }
237 }
238
239
240 private fun PsiDocTag.contentElements(): Iterable<PsiElement> {
241 val tagValueElements = children
242 .dropWhile { it.node?.elementType == JavaDocTokenType.DOC_TAG_NAME }
243 .dropWhile { it is PsiWhiteSpace }
244 .filterNot { it.node?.elementType == JavaDocTokenType.DOC_COMMENT_LEADING_ASTERISKS }
245 return if (getSubjectName() != null) tagValueElements.dropWhile { it is PsiDocTagValue } else tagValueElements
246 }
247
248 private fun ContentBlock.convertJavadocElements(elements: Iterable<PsiElement>, element: PsiNamedElement) {
249 val doc = Jsoup.parse(expandAllForElements(elements, element))
250 doc.body().childNodes().forEach {
251 convertHtmlNode(it)?.let { append(it) }
252 }
253 doc.head().childNodes().forEach {
254 convertHtmlNode(it)?.let { append(it) }
255 }
256 }
257
258 private fun ContentBlock.convertJavadocElementsToAttrDesc(elements: Iterable<PsiElement>, element: PsiNamedElement) {
259 val doc = Jsoup.parse(expandAllForElements(elements, element))
260 doc.body().childNodes().forEach {
261 convertHtmlNode(it)?.let {
262 var content = it
263 if (content is ContentText) {
264 var description = content.text
265 val matcher = TEXT.matcher(content.text)
266 if (matcher.matches()) {
267 val command = matcher.group(1)
268 if (command == "description") {
269 description = matcher.group(2)
270 content = ContentText(description)
271 }
272 }
273 }
274 append(content)
275 }
276 }
277 }
278
279 private fun expandAllForElements(elements: Iterable<PsiElement>, element: PsiNamedElement): String {
280 val htmlBuilder = StringBuilder()
281 elements.forEach {
282 if (it is PsiInlineDocTag) {
283 htmlBuilder.append(convertInlineDocTag(it, element))
284 } else {
285 htmlBuilder.append(it.text)
286 }
287 }
288 return htmlBuilder.toString().trim()
289 }
290
291 private fun convertHtmlNode(node: Node, isBlockCode: Boolean = false): ContentNode? {
292 if (isBlockCode) {
293 return if (node is TextNode) { // Fixes b/129762453
294 val codeNode = CodeNode(node.wholeText, "")
295 ContentText(codeNode.text().removePrefix("#"))
296 } else { // Fixes b/129857975
297 ContentText(node.toString())
298 }
299 }
300 if (node is TextNode) {
301 return ContentText(node.text().removePrefix("#"))
302 } else if (node is Element) {
303 val childBlock = createBlock(node)
304 node.childNodes().forEach {
305 val child = convertHtmlNode(it, isBlockCode = childBlock is ContentBlockCode)
306 if (child != null) {
307 childBlock.append(child)
308 }
309 }
310 return (childBlock)
311 }
312 return null
313 }
314
315 private fun createBlock(element: Element): ContentBlock = when (element.tagName()) {
316 "p" -> ContentParagraph()
317 "b", "strong" -> ContentStrong()
318 "i", "em" -> ContentEmphasis()
319 "s", "del" -> ContentStrikethrough()
320 "code" -> ContentCode()
321 "pre" -> ContentBlockCode()
322 "ul" -> ContentUnorderedList()
323 "ol" -> ContentOrderedList()
324 "li" -> ContentListItem()
325 "a" -> createLink(element)
326 "br" -> ContentBlock().apply { hardLineBreak() }
327
328 "dl" -> ContentDescriptionList()
329 "dt" -> ContentDescriptionTerm()
330 "dd" -> ContentDescriptionDefinition()
331
332 "table" -> ContentTable()
333 "tbody" -> ContentTableBody()
334 "tr" -> ContentTableRow()
335 "th" -> {
336 val colspan = element.attr("colspan")
337 val rowspan = element.attr("rowspan")
338 ContentTableHeader(colspan, rowspan)
339 }
340 "td" -> {
341 val colspan = element.attr("colspan")
342 val rowspan = element.attr("rowspan")
343 ContentTableCell(colspan, rowspan)
344 }
345
346 "h1" -> ContentHeading(1)
347 "h2" -> ContentHeading(2)
348 "h3" -> ContentHeading(3)
349 "h4" -> ContentHeading(4)
350 "h5" -> ContentHeading(5)
351 "h6" -> ContentHeading(6)
352
353 "div" -> {
354 val divClass = element.attr("class")
355 if (divClass == "special reference" || divClass == "note") ContentSpecialReference()
356 else ContentParagraph()
357 }
358
359 "script" -> {
360
361 // If the `type` attr is an empty string, we want to use null instead so that the resulting generated
362 // Javascript does not contain a `type` attr.
363 //
364 // Example:
365 // type == "" => <script type="" src="...">
366 // type == null => <script src="...">
367 val type = if (element.attr("type").isNotEmpty()) {
368 element.attr("type")
369 } else {
370 null
371 }
372 ScriptBlock(type, element.attr("src"))
373 }
374
375 else -> ContentBlock()
376 }
377
378 private fun createLink(element: Element): ContentBlock {
379 return when {
380 element.hasAttr("docref") -> {
381 val docref = element.attr("docref")
382 ContentNodeLazyLink(docref, { -> refGraph.lookupOrWarn(docref, logger) })
383 }
384 element.hasAttr("href") -> {
385 val href = element.attr("href")
386
387 val uri = try {
388 URI(href)
389 } catch (_: Exception) {
390 null
391 }
392
393 if (uri?.isAbsolute == false) {
394 ContentLocalLink(href)
395 } else {
396 ContentExternalLink(href)
397 }
398 }
399 element.hasAttr("name") -> {
400 ContentBookmark(element.attr("name"))
401 }
402 else -> ContentBlock()
403 }
404 }
405
406 private fun MutableContent.convertSeeTag(tag: PsiDocTag) {
407 val linkElement = tag.linkElement() ?: return
408 val seeSection = findSectionByTag(ContentTags.SeeAlso) ?: addSection(ContentTags.SeeAlso, null)
409
410 val valueElement = tag.referenceElement()
411 val externalLink = resolveExternalLink(valueElement)
412 val text = ContentText(linkElement.text)
413
414 val linkSignature by lazy { resolveInternalLink(valueElement) }
415 val node = when {
416 externalLink != null -> {
417 val linkNode = ContentExternalLink(externalLink)
418 linkNode.append(text)
419 linkNode
420 }
421 linkSignature != null -> {
422 @Suppress("USELESS_CAST")
423 val signature: String = linkSignature as String
424 val linkNode =
425 ContentNodeLazyLink(
426 (tag.valueElement ?: linkElement).text
427 ) { refGraph.lookupOrWarn(signature, logger) }
428 linkNode.append(text)
429 linkNode
430 }
431 else -> text
432 }
433 seeSection.append(node)
434 }
435
436 private fun convertInlineDocTag(tag: PsiInlineDocTag, element: PsiNamedElement) = when (tag.name) {
437 "link", "linkplain" -> {
438 val valueElement = tag.referenceElement()
439 val externalLink = resolveExternalLink(valueElement)
440 val linkSignature by lazy { resolveInternalLink(valueElement) }
441 if (externalLink != null || linkSignature != null) {
442
443 // sometimes `dataElements` contains multiple `PsiDocToken` elements and some have whitespace in them
444 // this is best effort to find the first non-empty one before falling back to using the symbol name.
445 val labelText = tag.dataElements.firstOrNull {
446 it is PsiDocToken && it.text?.trim()?.isNotEmpty() ?: false
447 }?.text ?: valueElement!!.text
448
449 val linkTarget = if (externalLink != null) "href=\"$externalLink\"" else "docref=\"$linkSignature\""
450 val link = "<a $linkTarget>$labelText</a>"
451 if (tag.name == "link") "<code>$link</code>" else link
452 } else if (valueElement != null) {
453 valueElement.text
454 } else {
455 ""
456 }
457 }
458 "code", "literal" -> {
459 val text = StringBuilder()
460 tag.dataElements.forEach { text.append(it.text) }
461 val escaped = text.toString().trimStart().htmlEscape()
462 if (tag.name == "code") "<code>$escaped</code>" else escaped
463 }
464 "inheritDoc" -> {
465 val result = (element as? PsiMethod)?.let {
466 // @{inheritDoc} is only allowed on functions
467 val parent = tag.parent
468 when (parent) {
469 is PsiDocComment -> element.findSuperDocCommentOrWarn()
470 is PsiDocTag -> element.findSuperDocTagOrWarn(parent)
471 else -> null
472 }
473 }
474 result ?: tag.text
475 }
476 "docRoot" -> {
477 // TODO: fix that
478 "https://developer.android.com/"
479 }
480 "sample" -> {
481 tag.text?.let { tagText ->
482 val (absolutePath, delimiter) = getSampleAnnotationInformation(tagText)
483 val code = retrieveCodeInFile(absolutePath, delimiter)
484 return if (code != null && code.isNotEmpty()) {
485 "<pre is-upgraded>$code</pre>"
486 } else {
487 ""
488 }
489 }
490 }
491
492 // Loads MathJax script from local source, which then updates MathJax HTML code
493 "usesMathJax" -> {
494 "<script src=\"/_static/js/managed/mathjax/MathJax.js?config=TeX-AMS_SVG\"></script>"
495 }
496
497 else -> tag.text
498 }
499
500 private fun PsiDocTag.referenceElement(): PsiElement? =
501 linkElement()?.let {
502 if (it.node.elementType == JavaDocElementType.DOC_REFERENCE_HOLDER) {
503 PsiTreeUtil.findChildOfType(it, PsiJavaCodeReferenceElement::class.java)
504 } else {
505 it
506 }
507 }
508
509 private fun PsiDocTag.linkElement(): PsiElement? =
510 valueElement ?: dataElements.firstOrNull { it !is PsiWhiteSpace }
511
512 private fun resolveExternalLink(valueElement: PsiElement?): String? {
513 val target = valueElement?.reference?.resolve()
514 if (target != null) {
515 return externalDocumentationLinkResolver.buildExternalDocumentationLink(target)
516 }
517 return null
518 }
519
520 private fun resolveInternalLink(valueElement: PsiElement?): String? {
521 val target = valueElement?.reference?.resolve()
522 if (target != null) {
523 return signatureProvider.signature(target)
524 }
525 return null
526 }
527
528 fun PsiDocTag.getSubjectName(): String? {
529 if (name == "param" || name == "throws" || name == "exception") {
530 return valueElement?.text
531 }
532 return null
533 }
534
535 private fun PsiMethod.findSuperDocCommentOrWarn(): String {
536 val method = findFirstSuperMethodWithDocumentation(this)
537 if (method != null) {
538 val descriptionElements = method.docComment?.descriptionElements?.dropWhile {
539 it.text.trim().isEmpty()
540 } ?: return ""
541
542 return expandAllForElements(descriptionElements, method)
543 }
544 logger.warn("No docs found on supertype with {@inheritDoc} method ${this.name} in ${this.containingFile.name}:${this.lineNumber()}")
545 return ""
546 }
547
548
549 private fun PsiMethod.findSuperDocTagOrWarn(elementToExpand: PsiDocTag): String {
550 val result = findFirstSuperMethodWithDocumentationforTag(elementToExpand, this)
551
552 if (result != null) {
553 val (method, tag) = result
554
555 val contentElements = tag.contentElements().dropWhile { it.text.trim().isEmpty() }
556
557 val expandedString = expandAllForElements(contentElements, method)
558
559 return expandedString
560 }
561 logger.warn("No docs found on supertype for @${elementToExpand.name} ${elementToExpand.getSubjectName()} with {@inheritDoc} method ${this.name} in ${this.containingFile.name}:${this.lineNumber()}")
562 return ""
563 }
564
565 private fun findFirstSuperMethodWithDocumentation(current: PsiMethod): PsiMethod? {
566 val superMethods = current.findSuperMethods()
567 for (method in superMethods) {
568 val docs = method.docComment?.descriptionElements?.dropWhile { it.text.trim().isEmpty() }
569 if (docs?.isNotEmpty() == true) {
570 return method
571 }
572 }
573 for (method in superMethods) {
574 val result = findFirstSuperMethodWithDocumentation(method)
575 if (result != null) {
576 return result
577 }
578 }
579
580 return null
581 }
582
583 private fun findFirstSuperMethodWithDocumentationforTag(
584 elementToExpand: PsiDocTag,
585 current: PsiMethod
586 ): Pair<PsiMethod, PsiDocTag>? {
587 val superMethods = current.findSuperMethods()
588 val mappedFilteredTags = superMethods.map {
589 it to it.docComment?.tags?.filter { it.name == elementToExpand.name }
590 }
591
592 for ((method, tags) in mappedFilteredTags) {
593 tags ?: continue
594 for (tag in tags) {
595 val (tagSubject, elementSubject) = when (tag.name) {
596 "throws" -> {
597 // match class names only for throws, ignore possibly fully qualified path
598 // TODO: Always match exactly here
599 tag.getSubjectName()?.split(".")?.last() to elementToExpand.getSubjectName()?.split(".")?.last()
600 }
601 else -> {
602 tag.getSubjectName() to elementToExpand.getSubjectName()
603 }
604 }
605
606 if (tagSubject == elementSubject) {
607 return method to tag
608 }
609 }
610 }
611
612 for (method in superMethods) {
613 val result = findFirstSuperMethodWithDocumentationforTag(elementToExpand, method)
614 if (result != null) {
615 return result
616 }
617 }
618 return null
619 }
620
621 /**
622 * Returns information inside @sample
623 *
624 * Component1 is the absolute path to the file
625 * Component2 is the delimiter if exists in the file
626 */
627 private fun getSampleAnnotationInformation(tagText: String): Pair<String, String> {
628 val pathContent = tagText
629 .trim { it == '{' || it == '}' }
630 .removePrefix("@sample ")
631
632 val formattedPath = pathContent.substringBefore(" ").trim()
633 val potentialDelimiter = pathContent.substringAfterLast(" ").trim()
634
635 val delimiter = if (potentialDelimiter == formattedPath) "" else potentialDelimiter
636 val path = "samples/$formattedPath"
637
638 return Pair(path, delimiter)
639 }
640
641 /**
642 * Retrieves the code inside a file.
643 *
644 * If betweenTag is not empty, it retrieves the code between
645 * BEGIN_INCLUDE($betweenTag) and END_INCLUDE($betweenTag) comments.
646 *
647 * Also, the method will trim every line with the number of spaces in the first line
648 */
649 private fun retrieveCodeInFile(path: String, betweenTag: String = "") = StringBuilder().apply {
650 try {
651 if (betweenTag.isEmpty()) {
652 appendContent(path)
653 } else {
654 appendContentBetweenIncludes(path, betweenTag)
655 }
656 } catch (e: java.lang.Exception) {
657 logger.error("No file found when processing Java @sample. Path to sample: $path\n")
658 }
659 }
660
661 private fun StringBuilder.appendContent(path: String) {
662 val spaces = InitialSpaceIndent()
663 File(path).forEachLine {
664 appendWithoutInitialIndent(it, spaces)
665 }
666 }
667
668 private fun StringBuilder.appendContentBetweenIncludes(path: String, includeTag: String) {
669 var shouldAppend = false
670 val beginning = "BEGIN_INCLUDE($includeTag)"
671 val end = "END_INCLUDE($includeTag)"
672 val spaces = InitialSpaceIndent()
673 File(path).forEachLine {
674 if (shouldAppend) {
675 if (it.contains(end)) {
676 shouldAppend = false
677 } else {
678 appendWithoutInitialIndent(it, spaces)
679 }
680 } else {
681 if (it.contains(beginning)) shouldAppend = true
682 }
683 }
684 }
685
686 private fun StringBuilder.appendWithoutInitialIndent(it: String, spaces: InitialSpaceIndent) {
687 if (spaces.value == -1) {
688 spaces.value = (it.length - it.trimStart().length).coerceAtLeast(0)
689 appendln(it)
690 } else {
691 appendln(if (it.isBlank()) it else it.substring(spaces.value, it.length))
692 }
693 }
694
695 private data class InitialSpaceIndent(var value: Int = -1)
696 }
697