# Copyright 2018 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import datetime import inspect import logging import json import endpoints from protorpc import messages from protorpc import remote from google.appengine.ext import ndb from webapp.src import vtslab_status as Status from webapp.src.proto import model MAX_QUERY_SIZE = 1000 COUNT_REQUEST_RESOURCE = endpoints.ResourceContainer(model.CountRequestMessage) GET_REQUEST_RESOURCE = endpoints.ResourceContainer(model.GetRequestMessage) class EndpointBase(remote.Service): """A base class for endpoint implementation.""" def GetCommonAttributes(self, resource, reference): """Gets a list of common attribute names. This method finds the attributes assigned in 'resource' instance, and filters out if the attributes are not a member of 'reference' class. Args: resource: either a protorpc.messages.Message instance, or a ndb.Model instance. reference: either a protorpc.messages.Message class, or a ndb.Model class. Returns: a list of string, attribute names exist on resource and reference. Raises: ValueError if resource or reference is not supported class. """ # check resource type and absorb list of assigned attributes. resource_attrs = self.GetAttributes(resource, assigned_only=True) reference_attrs = self.GetAttributes(reference) return [x for x in resource_attrs if x in reference_attrs] def GetAttributes(self, value, assigned_only=False): """Gets a list of attributes. Args: value: a class instance or a class itself. assigned_only: True to get only assigned attributes when value is an instance, False to get all attributes. Raises: ValueError if value is not supported class. """ attrs = [] if inspect.isclass(value): if assigned_only: logging.warning( "Please use a class instance for 'resource' argument.") if (issubclass(value, messages.Message) or issubclass(value, ndb.Model)): attrs = [ x[0] for x in value.__dict__.items() if not x[0].startswith("_") ] else: raise ValueError("Only protorpc.messages.Message or ndb.Model " "class are supported.") else: if isinstance(value, messages.Message): attrs = [ x.name for x in value.all_fields() if not assigned_only or ( value.get_assigned_value(x.name) not in [None, []]) ] elif isinstance(value, ndb.Model): attrs = [ x for x in list(value.to_dict()) if not assigned_only or ( getattr(value, x, None) not in [None, []]) ] else: raise ValueError("Only protorpc.messages.Message or ndb.Model " "class are supported.") return attrs def Count(self, metaclass, filters=None): """Counts entities from datastore with options. Args: metaclass: a metaclass for ndb model. filters: a list of tuples. Each tuple consists of three values: key, method, and value. Returns: a number of entities. """ query, _ = self.CreateQueryFilter(metaclass=metaclass, filters=filters) return query.count() def Fetch(self, metaclass, size, offset=0, filters=None, sort_key="", direction="asc"): """Fetches entities from datastore with options. Args: metaclass: a metaclass for ndb model. size: an integer, max number of entities to fetch at once. offset: an integer, number of query results to skip. filters: a list of filter tuple, a form of (key: string, method: integer, value: string). sort_key: a string, key name to sort by. direction: a string, "asc" for ascending order and "desc" for descending order. Returns: a list of fetched entities. a boolean, True if there is next page or False if not. """ query, empty_repeated_field = self.CreateQueryFilter( metaclass=metaclass, filters=filters) sorted_query = self.SortQuery( query=query, metaclass=metaclass, sort_key=sort_key, direction=direction) if size: entities, _, more = sorted_query.fetch_page( page_size=size, offset=offset) else: entities = sorted_query.fetch() more = False if empty_repeated_field: entities = [ x for x in entities if all([not getattr(x, attr) for attr in empty_repeated_field]) ] return entities, more def CreateQueryFilter(self, metaclass, filters): """Creates a query with the given filters. Args: metaclass: a metaclass for ndb model. filters: a list of tuples. Each tuple consists of three values: key, method, and value. Returns: a filtered query for the given metaclass. a list of strings that failed to create the query due to its empty value for the repeated property. """ empty_repeated_field = [] query = metaclass.query() if not filters: return query, empty_repeated_field for _filter in filters: property_key = _filter["key"] method = _filter["method"] value = _filter["value"] if type(value) is str or type(value) is unicode: if isinstance(metaclass._properties[property_key], ndb.BooleanProperty): value = value.lower() in ("yes", "true", "1") elif isinstance(metaclass._properties[property_key], ndb.IntegerProperty): value = int(value) if metaclass._properties[property_key]._repeated: if value: value = [value] if method == Status.FILTER_METHOD[Status.FILTER_Has]: query = query.filter( getattr(metaclass, property_key).IN(value)) else: logging.warning( "You cannot compare repeated " "properties except 'IN(has)' operation.") else: logging.debug("Empty repeated list cannot be queried.") empty_repeated_field.append(value) elif isinstance(metaclass._properties[property_key], ndb.DateTimeProperty): if method == Status.FILTER_METHOD[Status.FILTER_LessThan]: query = query.filter( getattr(metaclass, property_key) < datetime.datetime. now() - datetime.timedelta(hours=int(value))) elif method == Status.FILTER_METHOD[Status.FILTER_GreaterThan]: query = query.filter( getattr(metaclass, property_key) > datetime.datetime. now() - datetime.timedelta(hours=int(value))) else: logging.debug("DateTimeProperty only allows <=(less than) " "and >=(greater than) operation.") else: if method == Status.FILTER_METHOD[Status.FILTER_EqualTo]: query = query.filter( getattr(metaclass, property_key) == value) elif method == Status.FILTER_METHOD[Status.FILTER_LessThan]: query = query.filter( getattr(metaclass, property_key) < value) elif method == Status.FILTER_METHOD[Status.FILTER_GreaterThan]: query = query.filter( getattr(metaclass, property_key) > value) elif method == Status.FILTER_METHOD[ Status.FILTER_LessThanOrEqualTo]: query = query.filter( getattr(metaclass, property_key) <= value) elif method == Status.FILTER_METHOD[ Status.FILTER_GreaterThanOrEqualTo]: query = query.filter( getattr(metaclass, property_key) >= value) elif method == Status.FILTER_METHOD[Status.FILTER_NotEqualTo]: query = query.filter( getattr(metaclass, property_key) != value).order( getattr(metaclass, property_key), metaclass.key) elif method == Status.FILTER_METHOD[Status.FILTER_Has]: query = query.filter( getattr(metaclass, property_key).IN(value)).order( getattr(metaclass, property_key), metaclass.key) else: logging.warning( "{} is not supported filter method.".format(method)) return query, empty_repeated_field def SortQuery(self, query, metaclass, sort_key, direction): """Sorts the given query with sort_key and direction. Args: query: a ndb query to sort. metaclass: a metaclass for ndb model. sort_key: a string, key name to sort by. direction: a string, "asc" for ascending order and "desc" for descending order. """ if sort_key: if direction == "desc": query = query.order(-getattr(metaclass, sort_key)) else: query = query.order(getattr(metaclass, sort_key)) return query def CreateFilterList(self, filter_string, metaclass): """Creates a list of filters. Args: filter_string: a string, stringified JSON which contains 'key', 'method', 'value' to build filter information. metaclass: a metaclass for ndb model. Returns: a list of tuples where each tuple consists of three values: key, method, and value. """ model_properties = self.GetAttributes(metaclass) filters = [] if filter_string: filters = json.loads(filter_string) for _filter in filters: if _filter["key"] not in model_properties: filters.remove(_filter) return filters def Get(self, request, metaclass, message): """Handles a request through /get endpoints API to retrieves entities. Args: request: a request body message received through /get API. metaclass: a metaclass for ndb model. This method will fetch the 'metaclass' type of model from datastore. message: a Protocol RPC message class. Fetched entities will be converted to this message class instances. Returns: a list of fetched entities. a boolean, True if there is next page or False if not. """ size = request.size if request.size else MAX_QUERY_SIZE offset = request.offset if request.offset else 0 filters = self.CreateFilterList( filter_string=request.filter, metaclass=metaclass) entities, more = self.Fetch( metaclass=metaclass, size=size, filters=filters, offset=offset, sort_key=request.sort, direction=request.direction, ) return_list = [] for entity in entities: entity_dict = {} assigned_attributes = self.GetCommonAttributes( resource=entity, reference=message) for attr in assigned_attributes: entity_dict[attr] = getattr(entity, attr, None) if hasattr(message, "urlsafe_key"): entity_dict["urlsafe_key"] = entity.key.urlsafe() return_list.append(entity_dict) return return_list, more