1 /*
2  * Copyright 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.server.appsearch.external.localstorage.converter;
18 
19 import android.annotation.NonNull;
20 import android.app.appsearch.AppSearchSchema;
21 import android.app.appsearch.EmbeddingVector;
22 import android.app.appsearch.GenericDocument;
23 import android.app.appsearch.exceptions.AppSearchException;
24 import android.util.ArrayMap;
25 import android.util.ArraySet;
26 
27 import com.android.server.appsearch.external.localstorage.AppSearchConfig;
28 import com.android.server.appsearch.external.localstorage.util.PrefixUtil;
29 
30 import com.google.android.icing.proto.DocumentProto;
31 import com.google.android.icing.proto.DocumentProtoOrBuilder;
32 import com.google.android.icing.proto.PropertyProto;
33 import com.google.android.icing.proto.SchemaTypeConfigProto;
34 import com.google.protobuf.ByteString;
35 
36 import java.util.ArrayDeque;
37 import java.util.ArrayList;
38 import java.util.Collections;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.Objects;
42 import java.util.Queue;
43 import java.util.Set;
44 
45 /**
46  * Translates a {@link GenericDocument} into a {@link DocumentProto}.
47  *
48  * @hide
49  */
50 public final class GenericDocumentToProtoConverter {
51     private static final String[] EMPTY_STRING_ARRAY = new String[0];
52     private static final long[] EMPTY_LONG_ARRAY = new long[0];
53     private static final double[] EMPTY_DOUBLE_ARRAY = new double[0];
54     private static final boolean[] EMPTY_BOOLEAN_ARRAY = new boolean[0];
55     private static final byte[][] EMPTY_BYTES_ARRAY = new byte[0][0];
56     private static final GenericDocument[] EMPTY_DOCUMENT_ARRAY = new GenericDocument[0];
57     private static final EmbeddingVector[] EMPTY_EMBEDDING_ARRAY = new EmbeddingVector[0];
58 
GenericDocumentToProtoConverter()59     private GenericDocumentToProtoConverter() {}
60 
61     /** Converts a {@link GenericDocument} into a {@link DocumentProto}. */
62     @NonNull
63     @SuppressWarnings("unchecked")
toDocumentProto(@onNull GenericDocument document)64     public static DocumentProto toDocumentProto(@NonNull GenericDocument document) {
65         Objects.requireNonNull(document);
66         DocumentProto.Builder mProtoBuilder = DocumentProto.newBuilder();
67         mProtoBuilder
68                 .setUri(document.getId())
69                 .setSchema(document.getSchemaType())
70                 .setNamespace(document.getNamespace())
71                 .setScore(document.getScore())
72                 .setTtlMs(document.getTtlMillis())
73                 .setCreationTimestampMs(document.getCreationTimestampMillis());
74         ArrayList<String> keys = new ArrayList<>(document.getPropertyNames());
75         Collections.sort(keys);
76         for (int i = 0; i < keys.size(); i++) {
77             String name = keys.get(i);
78             PropertyProto.Builder propertyProto = PropertyProto.newBuilder().setName(name);
79             Object property = document.getProperty(name);
80             if (property instanceof String[]) {
81                 String[] stringValues = (String[]) property;
82                 for (int j = 0; j < stringValues.length; j++) {
83                     propertyProto.addStringValues(stringValues[j]);
84                 }
85             } else if (property instanceof long[]) {
86                 long[] longValues = (long[]) property;
87                 for (int j = 0; j < longValues.length; j++) {
88                     propertyProto.addInt64Values(longValues[j]);
89                 }
90             } else if (property instanceof double[]) {
91                 double[] doubleValues = (double[]) property;
92                 for (int j = 0; j < doubleValues.length; j++) {
93                     propertyProto.addDoubleValues(doubleValues[j]);
94                 }
95             } else if (property instanceof boolean[]) {
96                 boolean[] booleanValues = (boolean[]) property;
97                 for (int j = 0; j < booleanValues.length; j++) {
98                     propertyProto.addBooleanValues(booleanValues[j]);
99                 }
100             } else if (property instanceof byte[][]) {
101                 byte[][] bytesValues = (byte[][]) property;
102                 for (int j = 0; j < bytesValues.length; j++) {
103                     propertyProto.addBytesValues(ByteString.copyFrom(bytesValues[j]));
104                 }
105             } else if (property instanceof GenericDocument[]) {
106                 GenericDocument[] documentValues = (GenericDocument[]) property;
107                 for (int j = 0; j < documentValues.length; j++) {
108                     DocumentProto proto = toDocumentProto(documentValues[j]);
109                     propertyProto.addDocumentValues(proto);
110                 }
111             } else if (property instanceof EmbeddingVector[]) {
112                 EmbeddingVector[] embeddingValues = (EmbeddingVector[]) property;
113                 for (int j = 0; j < embeddingValues.length; j++) {
114                     propertyProto.addVectorValues(embeddingVectorToVectorProto(embeddingValues[j]));
115                 }
116             } else if (property == null) {
117                 throw new IllegalStateException(
118                         String.format("Property \"%s\" doesn't have any value!", name));
119             } else {
120                 throw new IllegalStateException(
121                         String.format(
122                                 "Property \"%s\" has unsupported value type %s",
123                                 name, property.getClass().toString()));
124             }
125             mProtoBuilder.addProperties(propertyProto);
126         }
127         return mProtoBuilder.build();
128     }
129 
130     /**
131      * Converts a {@link DocumentProto} into a {@link GenericDocument}.
132      *
133      * <p>In the case that the {@link DocumentProto} object proto has no values set, the converter
134      * searches for the matching property name in the {@link SchemaTypeConfigProto} object for the
135      * document, and infers the correct default value to set for the empty property based on the
136      * data type of the property defined by the schema type.
137      *
138      * @param proto the document to convert to a {@link GenericDocument} instance. The document
139      *     proto should have its package + database prefix stripped from its fields.
140      * @param prefix the package + database prefix used searching the {@code schemaTypeMap}.
141      * @param schemaTypeMap map of prefixed schema type to {@link SchemaTypeConfigProto}, used for
142      *     looking up the default empty value to set for a document property that has all empty
143      *     values.
144      */
145     @NonNull
toGenericDocument( @onNull DocumentProtoOrBuilder proto, @NonNull String prefix, @NonNull Map<String, SchemaTypeConfigProto> schemaTypeMap, @NonNull AppSearchConfig config)146     public static GenericDocument toGenericDocument(
147             @NonNull DocumentProtoOrBuilder proto,
148             @NonNull String prefix,
149             @NonNull Map<String, SchemaTypeConfigProto> schemaTypeMap,
150             @NonNull AppSearchConfig config)
151             throws AppSearchException {
152         Objects.requireNonNull(proto);
153         GenericDocument.Builder<?> documentBuilder =
154                 new GenericDocument.Builder<>(
155                                 proto.getNamespace(), proto.getUri(), proto.getSchema())
156                         .setScore(proto.getScore())
157                         .setTtlMillis(proto.getTtlMs())
158                         .setCreationTimestampMillis(proto.getCreationTimestampMs());
159         String prefixedSchemaType = prefix + proto.getSchema();
160         if (config.shouldRetrieveParentInfo()) {
161             List<String> parentSchemaTypes =
162                     getUnprefixedParentSchemaTypes(prefixedSchemaType, schemaTypeMap);
163             if (!parentSchemaTypes.isEmpty()) {
164                 if (config.shouldStoreParentInfoAsSyntheticProperty()) {
165                     documentBuilder.setPropertyString(
166                             GenericDocument.PARENT_TYPES_SYNTHETIC_PROPERTY,
167                             parentSchemaTypes.toArray(new String[0]));
168                 } else {
169                     documentBuilder.setParentTypes(parentSchemaTypes);
170                 }
171             }
172         }
173 
174         for (int i = 0; i < proto.getPropertiesCount(); i++) {
175             PropertyProto property = proto.getProperties(i);
176             String name = property.getName();
177             if (property.getStringValuesCount() > 0) {
178                 String[] values = new String[property.getStringValuesCount()];
179                 for (int j = 0; j < values.length; j++) {
180                     values[j] = property.getStringValues(j);
181                 }
182                 documentBuilder.setPropertyString(name, values);
183             } else if (property.getInt64ValuesCount() > 0) {
184                 long[] values = new long[property.getInt64ValuesCount()];
185                 for (int j = 0; j < values.length; j++) {
186                     values[j] = property.getInt64Values(j);
187                 }
188                 documentBuilder.setPropertyLong(name, values);
189             } else if (property.getDoubleValuesCount() > 0) {
190                 double[] values = new double[property.getDoubleValuesCount()];
191                 for (int j = 0; j < values.length; j++) {
192                     values[j] = property.getDoubleValues(j);
193                 }
194                 documentBuilder.setPropertyDouble(name, values);
195             } else if (property.getBooleanValuesCount() > 0) {
196                 boolean[] values = new boolean[property.getBooleanValuesCount()];
197                 for (int j = 0; j < values.length; j++) {
198                     values[j] = property.getBooleanValues(j);
199                 }
200                 documentBuilder.setPropertyBoolean(name, values);
201             } else if (property.getBytesValuesCount() > 0) {
202                 byte[][] values = new byte[property.getBytesValuesCount()][];
203                 for (int j = 0; j < values.length; j++) {
204                     values[j] = property.getBytesValues(j).toByteArray();
205                 }
206                 documentBuilder.setPropertyBytes(name, values);
207             } else if (property.getDocumentValuesCount() > 0) {
208                 GenericDocument[] values = new GenericDocument[property.getDocumentValuesCount()];
209                 for (int j = 0; j < values.length; j++) {
210                     values[j] =
211                             toGenericDocument(
212                                     property.getDocumentValues(j), prefix, schemaTypeMap, config);
213                 }
214                 documentBuilder.setPropertyDocument(name, values);
215             } else if (property.getVectorValuesCount() > 0) {
216                 EmbeddingVector[] values = new EmbeddingVector[property.getVectorValuesCount()];
217                 for (int j = 0; j < values.length; j++) {
218                     values[j] = vectorProtoToEmbeddingVector(property.getVectorValues(j));
219                 }
220                 documentBuilder.setPropertyEmbedding(name, values);
221             } else {
222                 // TODO(b/184966497): Optimize by caching PropertyConfigProto
223                 SchemaTypeConfigProto schema =
224                         Objects.requireNonNull(schemaTypeMap.get(prefixedSchemaType));
225                 setEmptyProperty(name, documentBuilder, schema);
226             }
227         }
228         return documentBuilder.build();
229     }
230 
231     /** Converts a {@link PropertyProto.VectorProto} into an {@link EmbeddingVector}. */
232     @NonNull
vectorProtoToEmbeddingVector( @onNull PropertyProto.VectorProto vectorProto)233     public static EmbeddingVector vectorProtoToEmbeddingVector(
234             @NonNull PropertyProto.VectorProto vectorProto) {
235         Objects.requireNonNull(vectorProto);
236 
237         float[] values = new float[vectorProto.getValuesCount()];
238         for (int i = 0; i < vectorProto.getValuesCount(); i++) {
239             values[i] = vectorProto.getValues(i);
240         }
241         return new EmbeddingVector(values, vectorProto.getModelSignature());
242     }
243 
244     /** Converts an {@link EmbeddingVector} into a {@link PropertyProto.VectorProto}. */
245     @NonNull
embeddingVectorToVectorProto( @onNull EmbeddingVector embedding)246     public static PropertyProto.VectorProto embeddingVectorToVectorProto(
247             @NonNull EmbeddingVector embedding) {
248         Objects.requireNonNull(embedding);
249 
250         PropertyProto.VectorProto.Builder builder = PropertyProto.VectorProto.newBuilder();
251         for (int i = 0; i < embedding.getValues().length; i++) {
252             builder.addValues(embedding.getValues()[i]);
253         }
254         builder.setModelSignature(embedding.getModelSignature());
255         return builder.build();
256     }
257 
258     /**
259      * Get the list of unprefixed parent type names of {@code prefixedSchemaType}.
260      *
261      * <p>It's guaranteed that child types always appear before parent types in the list.
262      */
263     // TODO(b/290389974): Consider caching the result based prefixedSchemaType, and reset the
264     //  cache whenever a new setSchema is called.
265     @NonNull
getUnprefixedParentSchemaTypes( @onNull String prefixedSchemaType, @NonNull Map<String, SchemaTypeConfigProto> schemaTypeMap)266     private static List<String> getUnprefixedParentSchemaTypes(
267             @NonNull String prefixedSchemaType,
268             @NonNull Map<String, SchemaTypeConfigProto> schemaTypeMap)
269             throws AppSearchException {
270         // Please note that neither DFS nor BFS order is guaranteed to always put child types
271         // before parent types (due to the diamond problem), so a topological sorting algorithm
272         // is required.
273         Map<String, Integer> inDegreeMap = new ArrayMap<>();
274         collectParentTypeInDegrees(
275                 prefixedSchemaType, schemaTypeMap, /* visited= */ new ArraySet<>(), inDegreeMap);
276 
277         List<String> result = new ArrayList<>();
278         Queue<String> queue = new ArrayDeque<>();
279         // prefixedSchemaType is the only type that has zero in-degree at this point.
280         queue.add(prefixedSchemaType);
281         while (!queue.isEmpty()) {
282             SchemaTypeConfigProto currentSchema =
283                     Objects.requireNonNull(schemaTypeMap.get(queue.poll()));
284             for (int i = 0; i < currentSchema.getParentTypesCount(); ++i) {
285                 String prefixedParentType = currentSchema.getParentTypes(i);
286                 int parentInDegree =
287                         Objects.requireNonNull(inDegreeMap.get(prefixedParentType)) - 1;
288                 inDegreeMap.put(prefixedParentType, parentInDegree);
289                 if (parentInDegree == 0) {
290                     result.add(PrefixUtil.removePrefix(prefixedParentType));
291                     queue.add(prefixedParentType);
292                 }
293             }
294         }
295         return result;
296     }
297 
collectParentTypeInDegrees( @onNull String prefixedSchemaType, @NonNull Map<String, SchemaTypeConfigProto> schemaTypeMap, @NonNull Set<String> visited, @NonNull Map<String, Integer> inDegreeMap)298     private static void collectParentTypeInDegrees(
299             @NonNull String prefixedSchemaType,
300             @NonNull Map<String, SchemaTypeConfigProto> schemaTypeMap,
301             @NonNull Set<String> visited,
302             @NonNull Map<String, Integer> inDegreeMap) {
303         if (visited.contains(prefixedSchemaType)) {
304             return;
305         }
306         visited.add(prefixedSchemaType);
307         SchemaTypeConfigProto schema =
308                 Objects.requireNonNull(schemaTypeMap.get(prefixedSchemaType));
309         for (int i = 0; i < schema.getParentTypesCount(); ++i) {
310             String prefixedParentType = schema.getParentTypes(i);
311             Integer parentInDegree = inDegreeMap.get(prefixedParentType);
312             if (parentInDegree == null) {
313                 parentInDegree = 0;
314             }
315             inDegreeMap.put(prefixedParentType, parentInDegree + 1);
316             collectParentTypeInDegrees(prefixedParentType, schemaTypeMap, visited, inDegreeMap);
317         }
318     }
319 
setEmptyProperty( @onNull String propertyName, @NonNull GenericDocument.Builder<?> documentBuilder, @NonNull SchemaTypeConfigProto schema)320     private static void setEmptyProperty(
321             @NonNull String propertyName,
322             @NonNull GenericDocument.Builder<?> documentBuilder,
323             @NonNull SchemaTypeConfigProto schema) {
324         @AppSearchSchema.PropertyConfig.DataType int dataType = 0;
325         for (int i = 0; i < schema.getPropertiesCount(); ++i) {
326             if (propertyName.equals(schema.getProperties(i).getPropertyName())) {
327                 dataType = schema.getProperties(i).getDataType().getNumber();
328                 break;
329             }
330         }
331 
332         switch (dataType) {
333             case AppSearchSchema.PropertyConfig.DATA_TYPE_STRING:
334                 documentBuilder.setPropertyString(propertyName, EMPTY_STRING_ARRAY);
335                 break;
336             case AppSearchSchema.PropertyConfig.DATA_TYPE_LONG:
337                 documentBuilder.setPropertyLong(propertyName, EMPTY_LONG_ARRAY);
338                 break;
339             case AppSearchSchema.PropertyConfig.DATA_TYPE_DOUBLE:
340                 documentBuilder.setPropertyDouble(propertyName, EMPTY_DOUBLE_ARRAY);
341                 break;
342             case AppSearchSchema.PropertyConfig.DATA_TYPE_BOOLEAN:
343                 documentBuilder.setPropertyBoolean(propertyName, EMPTY_BOOLEAN_ARRAY);
344                 break;
345             case AppSearchSchema.PropertyConfig.DATA_TYPE_BYTES:
346                 documentBuilder.setPropertyBytes(propertyName, EMPTY_BYTES_ARRAY);
347                 break;
348             case AppSearchSchema.PropertyConfig.DATA_TYPE_DOCUMENT:
349                 documentBuilder.setPropertyDocument(propertyName, EMPTY_DOCUMENT_ARRAY);
350                 break;
351             case AppSearchSchema.PropertyConfig.DATA_TYPE_EMBEDDING:
352                 documentBuilder.setPropertyEmbedding(propertyName, EMPTY_EMBEDDING_ARRAY);
353                 break;
354             default:
355                 throw new IllegalStateException("Unknown type of value: " + propertyName);
356         }
357     }
358 }
359