1#!/usr/bin/env python
2#*-* coding: utf-8
3# Copyright 2016 The Closure Linter Authors. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS-IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Closure typeannotation parsing and utilities."""
18
19
20
21from closure_linter import errors
22from closure_linter import javascripttokens
23from closure_linter.common import error
24
25# Shorthand
26TYPE = javascripttokens.JavaScriptTokenType
27
28
29class TypeAnnotation(object):
30  """Represents a structured view of a closure type annotation.
31
32  Attribute:
33    identifier: The name of the type.
34    key_type: The name part before a colon.
35    sub_types: The list of sub_types used e.g. for Array.<…>
36    or_null: The '?' annotation
37    not_null: The '!' annotation
38    type_group: If this a a grouping (a|b), but does not include function(a).
39    return_type: The return type of a function definition.
40    alias: The actual type set by closurizednamespaceinfo if the identifier uses
41        an alias to shorten the name.
42    tokens: An ordered list of tokens used for this type. May contain
43        TypeAnnotation instances for sub_types, key_type or return_type.
44  """
45
46  IMPLICIT_TYPE_GROUP = 2
47
48  NULLABILITY_UNKNOWN = 2
49
50  FUNCTION_TYPE = 'function'
51  NULL_TYPE = 'null'
52  VAR_ARGS_TYPE = '...'
53
54  # Frequently used known non-nullable types.
55  NON_NULLABLE = frozenset([
56      'boolean', FUNCTION_TYPE, 'number', 'string', 'undefined'])
57  # Frequently used known nullable types.
58  NULLABLE_TYPE_WHITELIST = frozenset([
59      'Array', 'Document', 'Element', 'Function', 'Node', 'NodeList',
60      'Object'])
61
62  def __init__(self):
63    self.identifier = ''
64    self.sub_types = []
65    self.or_null = False
66    self.not_null = False
67    self.type_group = False
68    self.alias = None
69    self.key_type = None
70    self.record_type = False
71    self.opt_arg = False
72    self.return_type = None
73    self.tokens = []
74
75  def IsFunction(self):
76    """Determines whether this is a function definition."""
77    return self.identifier == TypeAnnotation.FUNCTION_TYPE
78
79  def IsConstructor(self):
80    """Determines whether this is a function definition for a constructor."""
81    key_type = self.sub_types and self.sub_types[0].key_type
82    return self.IsFunction() and key_type.identifier == 'new'
83
84  def IsRecordType(self):
85    """Returns True if this type is a record type."""
86    return (self.record_type or
87            any(t.IsRecordType() for t in self.sub_types))
88
89  def IsVarArgsType(self):
90    """Determines if the type is a var_args type, i.e. starts with '...'."""
91    return self.identifier.startswith(TypeAnnotation.VAR_ARGS_TYPE) or (
92        self.type_group == TypeAnnotation.IMPLICIT_TYPE_GROUP and
93        self.sub_types[0].identifier.startswith(TypeAnnotation.VAR_ARGS_TYPE))
94
95  def IsEmpty(self):
96    """Returns True if the type is empty."""
97    return not self.tokens
98
99  def IsUnknownType(self):
100    """Returns True if this is the unknown type {?}."""
101    return (self.or_null
102            and not self.identifier
103            and not self.sub_types
104            and not self.return_type)
105
106  def Append(self, item):
107    """Adds a sub_type to this type and finalizes it.
108
109    Args:
110      item: The TypeAnnotation item to append.
111    """
112    # item is a TypeAnnotation instance, so pylint: disable=protected-access
113    self.sub_types.append(item._Finalize(self))
114
115  def __repr__(self):
116    """Reconstructs the type definition."""
117    append = ''
118    if self.sub_types:
119      separator = (',' if not self.type_group else '|')
120      if self.IsFunction():
121        surround = '(%s)'
122      else:
123        surround = {False: '{%s}' if self.record_type else '<%s>',
124                    True: '(%s)',
125                    TypeAnnotation.IMPLICIT_TYPE_GROUP: '%s'}[self.type_group]
126      append = surround % separator.join(repr(t) for t in self.sub_types)
127    if self.return_type:
128      append += ':%s' % repr(self.return_type)
129    append += '=' if self.opt_arg else ''
130    prefix = '' + ('?' if self.or_null else '') + ('!' if self.not_null else '')
131    keyword = '%s:' % repr(self.key_type) if self.key_type else ''
132    return keyword + prefix + '%s' % (self.alias or self.identifier) + append
133
134  def ToString(self):
135    """Concats the type's tokens to form a string again."""
136    ret = []
137    for token in self.tokens:
138      if not isinstance(token, TypeAnnotation):
139        ret.append(token.string)
140      else:
141        ret.append(token.ToString())
142    return ''.join(ret)
143
144  def Dump(self, indent=''):
145    """Dumps this type's structure for debugging purposes."""
146    result = []
147    for t in self.tokens:
148      if isinstance(t, TypeAnnotation):
149        result.append(indent + str(t) + ' =>\n' + t.Dump(indent + '  '))
150      else:
151        result.append(indent + str(t))
152    return '\n'.join(result)
153
154  def IterIdentifiers(self):
155    """Iterates over all identifiers in this type and its subtypes."""
156    if self.identifier:
157      yield self.identifier
158    for subtype in self.IterTypes():
159      for identifier in subtype.IterIdentifiers():
160        yield identifier
161
162  def IterTypeGroup(self):
163    """Iterates over all types in the type group including self.
164
165    Yields:
166      If this is a implicit or manual type-group: all sub_types.
167      Otherwise: self
168      E.g. for @type {Foo.<Bar>} this will yield only Foo.<Bar>,
169      for @type {Foo|(Bar|Sample)} this will yield Foo, Bar and Sample.
170
171    """
172    if self.type_group:
173      for sub_type in self.sub_types:
174        for sub_type in sub_type.IterTypeGroup():
175          yield sub_type
176    else:
177      yield self
178
179  def IterTypes(self):
180    """Iterates over each subtype as well as return and key types."""
181    if self.return_type:
182      yield self.return_type
183
184    if self.key_type:
185      yield self.key_type
186
187    for sub_type in self.sub_types:
188      yield sub_type
189
190  def GetNullability(self, modifiers=True):
191    """Computes whether the type may be null.
192
193    Args:
194      modifiers: Whether the modifiers ? and ! should be considered in the
195                 evaluation.
196    Returns:
197      True if the type allows null, False if the type is strictly non nullable
198      and NULLABILITY_UNKNOWN if the nullability cannot be determined.
199    """
200
201    # Explicitly marked nullable types or 'null' are nullable.
202    if ((modifiers and self.or_null) or
203        self.identifier == TypeAnnotation.NULL_TYPE):
204      return True
205
206    # Explicitly marked non-nullable types or non-nullable base types:
207    if ((modifiers and self.not_null) or self.record_type
208        or self.identifier in TypeAnnotation.NON_NULLABLE):
209      return False
210
211    # A type group is nullable if any of its elements are nullable.
212    if self.type_group:
213      maybe_nullable = False
214      for sub_type in self.sub_types:
215        nullability = sub_type.GetNullability()
216        if nullability == self.NULLABILITY_UNKNOWN:
217          maybe_nullable = nullability
218        elif nullability:
219          return True
220      return maybe_nullable
221
222    # Whitelisted types are nullable.
223    if self.identifier.rstrip('.') in TypeAnnotation.NULLABLE_TYPE_WHITELIST:
224      return True
225
226    # All other types are unknown (most should be nullable, but
227    # enums are not and typedefs might not be).
228    return TypeAnnotation.NULLABILITY_UNKNOWN
229
230  def WillAlwaysBeNullable(self):
231    """Computes whether the ! flag is illegal for this type.
232
233    This is the case if this type or any of the subtypes is marked as
234    explicitly nullable.
235
236    Returns:
237      True if the ! flag would be illegal.
238    """
239    if self.or_null or self.identifier == TypeAnnotation.NULL_TYPE:
240      return True
241
242    if self.type_group:
243      return any(t.WillAlwaysBeNullable() for t in self.sub_types)
244
245    return False
246
247  def _Finalize(self, parent):
248    """Fixes some parsing issues once the TypeAnnotation is complete."""
249
250    # Normalize functions whose definition ended up in the key type because
251    # they defined a return type after a colon.
252    if (self.key_type and
253        self.key_type.identifier == TypeAnnotation.FUNCTION_TYPE):
254      current = self.key_type
255      current.return_type = self
256      self.key_type = None
257      # opt_arg never refers to the return type but to the function itself.
258      current.opt_arg = self.opt_arg
259      self.opt_arg = False
260      return current
261
262    # If a typedef just specified the key, it will not end up in the key type.
263    if parent.record_type and not self.key_type:
264      current = TypeAnnotation()
265      current.key_type = self
266      current.tokens.append(self)
267      return current
268    return self
269
270  def FirstToken(self):
271    """Returns the first token used in this type or any of its subtypes."""
272    first = self.tokens[0]
273    return first.FirstToken() if isinstance(first, TypeAnnotation) else first
274
275
276def Parse(token, token_end, error_handler):
277  """Parses a type annotation and returns a TypeAnnotation object."""
278  return TypeAnnotationParser(error_handler).Parse(token.next, token_end)
279
280
281class TypeAnnotationParser(object):
282  """A parser for type annotations constructing the TypeAnnotation object."""
283
284  def __init__(self, error_handler):
285    self._stack = []
286    self._error_handler = error_handler
287    self._closing_error = False
288
289  def Parse(self, token, token_end):
290    """Parses a type annotation and returns a TypeAnnotation object."""
291    root = TypeAnnotation()
292    self._stack.append(root)
293    current = TypeAnnotation()
294    root.tokens.append(current)
295
296    while token and token != token_end:
297      if token.type in (TYPE.DOC_TYPE_START_BLOCK, TYPE.DOC_START_BRACE):
298        if token.string == '(':
299          if current.identifier and current.identifier not in [
300              TypeAnnotation.FUNCTION_TYPE, TypeAnnotation.VAR_ARGS_TYPE]:
301            self.Error(token,
302                       'Invalid identifier for (): "%s"' % current.identifier)
303          current.type_group = (
304              current.identifier != TypeAnnotation.FUNCTION_TYPE)
305        elif token.string == '{':
306          current.record_type = True
307        current.tokens.append(token)
308        self._stack.append(current)
309        current = TypeAnnotation()
310        self._stack[-1].tokens.append(current)
311
312      elif token.type in (TYPE.DOC_TYPE_END_BLOCK, TYPE.DOC_END_BRACE):
313        prev = self._stack.pop()
314        prev.Append(current)
315        current = prev
316
317        # If an implicit type group was created, close it as well.
318        if prev.type_group == TypeAnnotation.IMPLICIT_TYPE_GROUP:
319          prev = self._stack.pop()
320          prev.Append(current)
321          current = prev
322        current.tokens.append(token)
323
324      elif token.type == TYPE.DOC_TYPE_MODIFIER:
325        if token.string == '!':
326          current.tokens.append(token)
327          current.not_null = True
328        elif token.string == '?':
329          current.tokens.append(token)
330          current.or_null = True
331        elif token.string == ':':
332          current.tokens.append(token)
333          prev = current
334          current = TypeAnnotation()
335          prev.tokens.append(current)
336          current.key_type = prev
337        elif token.string == '=':
338          # For implicit type groups the '=' refers to the parent.
339          try:
340            if self._stack[-1].type_group == TypeAnnotation.IMPLICIT_TYPE_GROUP:
341              self._stack[-1].tokens.append(token)
342              self._stack[-1].opt_arg = True
343            else:
344              current.tokens.append(token)
345              current.opt_arg = True
346          except IndexError:
347            self.ClosingError(token)
348        elif token.string == '|':
349          # If a type group has explicitly been opened, do a normal append.
350          # Otherwise we have to open the type group and move the current
351          # type into it, before appending
352          if not self._stack[-1].type_group:
353            type_group = TypeAnnotation()
354            if (current.key_type and
355                current.key_type.identifier != TypeAnnotation.FUNCTION_TYPE):
356              type_group.key_type = current.key_type
357              current.key_type = None
358            type_group.type_group = TypeAnnotation.IMPLICIT_TYPE_GROUP
359            # Fix the token order
360            prev = self._stack[-1].tokens.pop()
361            self._stack[-1].tokens.append(type_group)
362            type_group.tokens.append(prev)
363            self._stack.append(type_group)
364          self._stack[-1].tokens.append(token)
365          self.Append(current, error_token=token)
366          current = TypeAnnotation()
367          self._stack[-1].tokens.append(current)
368        elif token.string == ',':
369          self.Append(current, error_token=token)
370          current = TypeAnnotation()
371          self._stack[-1].tokens.append(token)
372          self._stack[-1].tokens.append(current)
373        else:
374          current.tokens.append(token)
375          self.Error(token, 'Invalid token')
376
377      elif token.type == TYPE.COMMENT:
378        current.tokens.append(token)
379        current.identifier += token.string.strip()
380
381      elif token.type in [TYPE.DOC_PREFIX, TYPE.WHITESPACE]:
382        current.tokens.append(token)
383
384      else:
385        current.tokens.append(token)
386        self.Error(token, 'Unexpected token')
387
388      token = token.next
389
390    self.Append(current, error_token=token)
391    try:
392      ret = self._stack.pop()
393    except IndexError:
394      self.ClosingError(token)
395      # The type is screwed up, but let's return something.
396      return current
397
398    if self._stack and (len(self._stack) != 1 or
399                        ret.type_group != TypeAnnotation.IMPLICIT_TYPE_GROUP):
400      self.Error(token, 'Too many opening items.')
401
402    return ret if len(ret.sub_types) > 1 else ret.sub_types[0]
403
404  def Append(self, type_obj, error_token):
405    """Appends a new TypeAnnotation object to the current parent."""
406    if self._stack:
407      self._stack[-1].Append(type_obj)
408    else:
409      self.ClosingError(error_token)
410
411  def ClosingError(self, token):
412    """Reports an error about too many closing items, but only once."""
413    if not self._closing_error:
414      self._closing_error = True
415      self.Error(token, 'Too many closing items.')
416
417  def Error(self, token, message):
418    """Calls the error_handler to post an error message."""
419    if self._error_handler:
420      self._error_handler.HandleError(error.Error(
421          errors.JSDOC_DOES_NOT_PARSE,
422          'Error parsing jsdoc type at token "%s" (column: %d): %s' %
423          (token.string, token.start_index, message), token))
424
425