1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.example.android.autofillkeyboard; 18 19 import static android.util.TypedValue.COMPLEX_UNIT_DIP; 20 21 import android.graphics.Color; 22 import android.graphics.drawable.Icon; 23 import android.inputmethodservice.InputMethodService; 24 import android.os.Bundle; 25 import android.os.Handler; 26 import android.os.Looper; 27 import android.util.Log; 28 import android.util.Size; 29 import android.util.TypedValue; 30 import android.view.LayoutInflater; 31 import android.view.View; 32 import android.view.ViewGroup; 33 import android.view.inputmethod.EditorInfo; 34 import android.widget.inline.InlineContentView; 35 import android.widget.inline.InlinePresentationSpec; 36 import android.view.inputmethod.InlineSuggestion; 37 import android.view.inputmethod.InlineSuggestionsRequest; 38 import android.view.inputmethod.InlineSuggestionsResponse; 39 import android.widget.Toast; 40 41 import androidx.autofill.inline.UiVersions; 42 import androidx.autofill.inline.UiVersions.StylesBuilder; 43 import androidx.autofill.inline.common.ImageViewStyle; 44 import androidx.autofill.inline.common.TextViewStyle; 45 import androidx.autofill.inline.common.ViewStyle; 46 import androidx.autofill.inline.v1.InlineSuggestionUi; 47 import androidx.autofill.inline.v1.InlineSuggestionUi.Style; 48 49 import java.util.ArrayList; 50 import java.util.Collections; 51 import java.util.List; 52 import java.util.Map; 53 import java.util.TreeMap; 54 import java.util.concurrent.ExecutorService; 55 import java.util.concurrent.Executors; 56 57 /** The {@link InputMethodService} implementation for Autofill keyboard. */ 58 public class AutofillImeService extends InputMethodService { 59 private static final boolean SHOWCASE_BG_FG_TRANSITION = false; 60 // To test this you need to change KeyboardArea style layout_height to 400dp 61 private static final boolean SHOWCASE_UP_DOWN_TRANSITION = false; 62 63 private static final long MOVE_SUGGESTIONS_TO_BG_TIMEOUT = 5000; 64 private static final long MOVE_SUGGESTIONS_TO_FG_TIMEOUT = 15000; 65 66 private static final long MOVE_SUGGESTIONS_UP_TIMEOUT = 5000; 67 private static final long MOVE_SUGGESTIONS_DOWN_TIMEOUT = 10000; 68 69 private InputView mInputView; 70 private Keyboard mKeyboard; 71 private Decoder mDecoder; 72 73 private ViewGroup mSuggestionStrip; 74 private ViewGroup mPinnedSuggestionsStart; 75 private ViewGroup mPinnedSuggestionsEnd; 76 private InlineContentClipView mScrollableSuggestionsClip; 77 private ViewGroup mScrollableSuggestions; 78 79 private final Handler mHandler = new Handler(Looper.getMainLooper()); 80 81 private final Runnable mMoveScrollableSuggestionsToBg = () -> { 82 mScrollableSuggestionsClip.setZOrderedOnTop(false); 83 Toast.makeText(AutofillImeService.this, "Chips moved to bg - not clickable", 84 Toast.LENGTH_SHORT).show(); 85 }; 86 87 private final Runnable mMoveScrollableSuggestionsToFg = () -> { 88 mScrollableSuggestionsClip.setZOrderedOnTop(true); 89 Toast.makeText(AutofillImeService.this, "Chips moved to fg - clickable", 90 Toast.LENGTH_SHORT).show(); 91 }; 92 93 private final Runnable mMoveScrollableSuggestionsUp = () -> { 94 mSuggestionStrip.animate().translationY(-50).setDuration(500).start(); 95 Toast.makeText(AutofillImeService.this, "Animating up", 96 Toast.LENGTH_SHORT).show(); 97 }; 98 99 private final Runnable mMoveScrollableSuggestionsDown = () -> { 100 mSuggestionStrip.animate().translationY(0).setDuration(500).start(); 101 Toast.makeText(AutofillImeService.this, "Animating down", 102 Toast.LENGTH_SHORT).show(); 103 }; 104 105 private ResponseState mResponseState = ResponseState.RESET; 106 private Runnable mDelayedDeletion; 107 private Runnable mPendingResponse; 108 109 @Override onCreateInputView()110 public View onCreateInputView() { 111 mInputView = (InputView) LayoutInflater.from(this).inflate(R.layout.input_view, null); 112 mKeyboard = Keyboard.qwerty(this); 113 mInputView.addView(mKeyboard.inflateKeyboardView(LayoutInflater.from(this), mInputView)); 114 mSuggestionStrip = mInputView.findViewById(R.id.suggestion_strip); 115 mPinnedSuggestionsStart = mInputView.findViewById(R.id.pinned_suggestions_start); 116 mPinnedSuggestionsEnd = mInputView.findViewById(R.id.pinned_suggestions_end); 117 mScrollableSuggestionsClip = mInputView.findViewById(R.id.scrollable_suggestions_clip); 118 mScrollableSuggestions = mInputView.findViewById(R.id.scrollable_suggestions); 119 return mInputView; 120 } 121 122 @Override onStartInput(EditorInfo attribute, boolean restarting)123 public void onStartInput(EditorInfo attribute, boolean restarting) { 124 super.onStartInput(attribute, restarting); 125 mDecoder = new Decoder(getCurrentInputConnection()); 126 if(mKeyboard != null) { 127 mKeyboard.reset(); 128 } 129 if (mResponseState == ResponseState.RECEIVE_RESPONSE) { 130 mResponseState = ResponseState.START_INPUT; 131 } else { 132 mResponseState = ResponseState.RESET; 133 } 134 } 135 136 @Override onFinishInput()137 public void onFinishInput() { 138 super.onFinishInput(); 139 } 140 cancelPendingResponse()141 private void cancelPendingResponse() { 142 if (mPendingResponse != null) { 143 Log.d(TAG, "Canceling pending response"); 144 mHandler.removeCallbacks(mPendingResponse); 145 mPendingResponse = null; 146 } 147 } 148 postPendingResponse(InlineSuggestionsResponse response)149 private void postPendingResponse(InlineSuggestionsResponse response) { 150 cancelPendingResponse(); 151 final List<InlineSuggestion> inlineSuggestions = response.getInlineSuggestions(); 152 mResponseState = ResponseState.RECEIVE_RESPONSE; 153 mPendingResponse = () -> { 154 mPendingResponse = null; 155 if (mResponseState == ResponseState.START_INPUT && inlineSuggestions.isEmpty()) { 156 scheduleDelayedDeletion(); 157 } else { 158 inflateThenShowSuggestions(inlineSuggestions); 159 } 160 mResponseState = ResponseState.RESET; 161 }; 162 mHandler.post(mPendingResponse); 163 } 164 cancelDelayedDeletion(String msg)165 private void cancelDelayedDeletion(String msg) { 166 if(mDelayedDeletion != null) { 167 Log.d(TAG, msg + " canceling delayed deletion"); 168 mHandler.removeCallbacks(mDelayedDeletion); 169 mDelayedDeletion = null; 170 } 171 } 172 scheduleDelayedDeletion()173 private void scheduleDelayedDeletion() { 174 if (mInputView != null && mDelayedDeletion == null) { 175 // We delay the deletion of the suggestions from previous input connection, to avoid 176 // the flicker caused by deleting them and immediately showing new suggestions for 177 // the current input connection. 178 Log.d(TAG, "Scheduling a delayed deletion of inline suggestions"); 179 mDelayedDeletion = () -> { 180 Log.d(TAG, "Executing scheduled deleting inline suggestions"); 181 mDelayedDeletion = null; 182 clearInlineSuggestionStrip(); 183 }; 184 mHandler.postDelayed(mDelayedDeletion, 200); 185 } 186 } 187 clearInlineSuggestionStrip()188 private void clearInlineSuggestionStrip() { 189 if (mInputView != null) { 190 updateInlineSuggestionStrip(Collections.emptyList()); 191 } 192 } 193 194 @Override onStartInputView(EditorInfo info, boolean restarting)195 public void onStartInputView(EditorInfo info, boolean restarting) { 196 super.onStartInputView(info, restarting); 197 } 198 199 @Override onFinishInputView(boolean finishingInput)200 public void onFinishInputView(boolean finishingInput) { 201 super.onFinishInputView(finishingInput); 202 if (!finishingInput) { 203 // This runs when the IME is hide (but not finished). We need to clear the suggestions. 204 // Otherwise, they will stay on the screen for a bit after the IME window disappears. 205 // TODO: right now the framework resends the suggestions when onStartInputView is 206 // called. If the framework is changed to not resend, then we need to cache the 207 // inline suggestion views locally and re-attach them when the IME is shown again by 208 // onStartInputView. 209 clearInlineSuggestionStrip(); 210 } 211 } 212 213 @Override onComputeInsets(Insets outInsets)214 public void onComputeInsets(Insets outInsets) { 215 super.onComputeInsets(outInsets); 216 if (mInputView != null) { 217 outInsets.contentTopInsets += mInputView.getTopInsets(); 218 } 219 outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT; 220 } 221 222 /***************** Inline Suggestions Demo Code *****************/ 223 224 private static final String TAG = "AutofillImeService"; 225 226 @Override onCreateInlineSuggestionsRequest(Bundle uiExtras)227 public InlineSuggestionsRequest onCreateInlineSuggestionsRequest(Bundle uiExtras) { 228 Log.d(TAG, "onCreateInlineSuggestionsRequest() called"); 229 StylesBuilder stylesBuilder = UiVersions.newStylesBuilder(); 230 Style style = InlineSuggestionUi.newStyleBuilder() 231 .setSingleIconChipStyle( 232 new ViewStyle.Builder() 233 .setBackground( 234 Icon.createWithResource(this, R.drawable.chip_background)) 235 .setPadding(0, 0, 0, 0) 236 .build()) 237 .setChipStyle( 238 new ViewStyle.Builder() 239 .setBackground( 240 Icon.createWithResource(this, R.drawable.chip_background)) 241 .setPadding(toPixel(5 + 8), 0, toPixel(5 + 8), 0) 242 .build()) 243 .setStartIconStyle(new ImageViewStyle.Builder().setLayoutMargin(0, 0, 0, 0).build()) 244 .setTitleStyle( 245 new TextViewStyle.Builder() 246 .setLayoutMargin(toPixel(4), 0, toPixel(4), 0) 247 .setTextColor(Color.parseColor("#FF202124")) 248 .setTextSize(16) 249 .build()) 250 .setSubtitleStyle( 251 new TextViewStyle.Builder() 252 .setLayoutMargin(0, 0, toPixel(4), 0) 253 .setTextColor(Color.parseColor("#99202124")) // 60% opacity 254 .setTextSize(14) 255 .build()) 256 .setEndIconStyle(new ImageViewStyle.Builder().setLayoutMargin(0, 0, 0, 0).build()) 257 .build(); 258 stylesBuilder.addStyle(style); 259 Bundle stylesBundle = stylesBuilder.build(); 260 261 final ArrayList<InlinePresentationSpec> presentationSpecs = new ArrayList<>(); 262 presentationSpecs.add(new InlinePresentationSpec.Builder(new Size(100, getHeight()), 263 new Size(740, getHeight())).setStyle(stylesBundle).build()); 264 presentationSpecs.add(new InlinePresentationSpec.Builder(new Size(100, getHeight()), 265 new Size(740, getHeight())).setStyle(stylesBundle).build()); 266 267 return new InlineSuggestionsRequest.Builder(presentationSpecs) 268 .setMaxSuggestionCount(6) 269 .build(); 270 } 271 toPixel(int dp)272 private int toPixel(int dp) { 273 return (int) TypedValue.applyDimension(COMPLEX_UNIT_DIP, dp, 274 getResources().getDisplayMetrics()); 275 } 276 getHeight()277 private int getHeight() { 278 return getResources().getDimensionPixelSize(R.dimen.keyboard_header_height); 279 } 280 281 @Override onInlineSuggestionsResponse(InlineSuggestionsResponse response)282 public boolean onInlineSuggestionsResponse(InlineSuggestionsResponse response) { 283 Log.d(TAG, 284 "onInlineSuggestionsResponse() called: " + response.getInlineSuggestions().size()); 285 cancelDelayedDeletion("onInlineSuggestionsResponse"); 286 postPendingResponse(response); 287 return true; 288 } 289 updateInlineSuggestionStrip(List<SuggestionItem> suggestionItems)290 private void updateInlineSuggestionStrip(List<SuggestionItem> suggestionItems) { 291 Log.d(TAG, "Actually updating the suggestion strip: " + suggestionItems.size()); 292 mPinnedSuggestionsStart.removeAllViews(); 293 mScrollableSuggestions.removeAllViews(); 294 mPinnedSuggestionsEnd.removeAllViews(); 295 296 if (suggestionItems.isEmpty()) { 297 return; 298 } 299 300 // TODO: refactor me 301 mScrollableSuggestionsClip.setBackgroundColor( 302 getColor(R.color.suggestion_strip_background)); 303 mSuggestionStrip.setVisibility(View.VISIBLE); 304 305 for (SuggestionItem suggestionItem : suggestionItems) { 306 if (suggestionItem == null) { 307 continue; 308 } 309 final InlineContentView suggestionView = suggestionItem.mView; 310 if (suggestionItem.mIsPinned) { 311 if (mPinnedSuggestionsStart.getChildCount() <= 0) { 312 mPinnedSuggestionsStart.addView(suggestionView); 313 } else { 314 mPinnedSuggestionsEnd.addView(suggestionView); 315 } 316 } else { 317 mScrollableSuggestions.addView(suggestionView); 318 } 319 } 320 321 if (SHOWCASE_BG_FG_TRANSITION) { 322 rescheduleShowcaseBgFgTransitions(); 323 } 324 if (SHOWCASE_UP_DOWN_TRANSITION) { 325 rescheduleShowcaseUpDownTransitions(); 326 } 327 } 328 rescheduleShowcaseBgFgTransitions()329 private void rescheduleShowcaseBgFgTransitions() { 330 final Handler handler = mInputView.getHandler(); 331 handler.removeCallbacks(mMoveScrollableSuggestionsToBg); 332 handler.postDelayed(mMoveScrollableSuggestionsToBg, MOVE_SUGGESTIONS_TO_BG_TIMEOUT); 333 handler.removeCallbacks(mMoveScrollableSuggestionsToFg); 334 handler.postDelayed(mMoveScrollableSuggestionsToFg, MOVE_SUGGESTIONS_TO_FG_TIMEOUT); 335 } 336 rescheduleShowcaseUpDownTransitions()337 private void rescheduleShowcaseUpDownTransitions() { 338 final Handler handler = mInputView.getHandler(); 339 handler.removeCallbacks(mMoveScrollableSuggestionsUp); 340 handler.postDelayed(mMoveScrollableSuggestionsUp, MOVE_SUGGESTIONS_UP_TIMEOUT); 341 handler.removeCallbacks(mMoveScrollableSuggestionsDown); 342 handler.postDelayed(mMoveScrollableSuggestionsDown, MOVE_SUGGESTIONS_DOWN_TIMEOUT); 343 } 344 inflateThenShowSuggestions( List<InlineSuggestion> inlineSuggestions)345 private void inflateThenShowSuggestions( List<InlineSuggestion> inlineSuggestions) { 346 final int totalSuggestionsCount = inlineSuggestions.size(); 347 if (inlineSuggestions.isEmpty()) { 348 // clear the suggestions and then return 349 getMainExecutor().execute(() -> updateInlineSuggestionStrip(Collections.EMPTY_LIST)); 350 return; 351 } 352 353 final Map<Integer, SuggestionItem> suggestionMap = Collections.synchronizedMap(( 354 new TreeMap<>())); 355 final ExecutorService executor = Executors.newSingleThreadExecutor(); 356 357 for (int i = 0; i < totalSuggestionsCount; i++) { 358 final int index = i; 359 final InlineSuggestion inlineSuggestion = inlineSuggestions.get(i); 360 final Size size = new Size(ViewGroup.LayoutParams.WRAP_CONTENT, 361 ViewGroup.LayoutParams.WRAP_CONTENT); 362 363 inlineSuggestion.inflate(this, size, executor, suggestionView -> { 364 Log.d(TAG, "new inline suggestion view ready"); 365 if(suggestionView != null) { 366 suggestionView.setOnClickListener((v) -> { 367 Log.d(TAG, "Received click on the suggestion"); 368 }); 369 suggestionView.setOnLongClickListener((v) -> { 370 Log.d(TAG, "Received long click on the suggestion"); 371 return true; 372 }); 373 final SuggestionItem suggestionItem = new SuggestionItem( 374 suggestionView, /*isAction*/ inlineSuggestion.getInfo().isPinned()); 375 suggestionMap.put(index, suggestionItem); 376 } else { 377 suggestionMap.put(index, null); 378 } 379 380 // Update the UI once the last inflation completed 381 if (suggestionMap.size() >= totalSuggestionsCount) { 382 final ArrayList<SuggestionItem> suggestionItems = new ArrayList<>( 383 suggestionMap.values()); 384 getMainExecutor().execute(() -> updateInlineSuggestionStrip(suggestionItems)); 385 } 386 }); 387 } 388 } 389 handle(String data)390 void handle(String data) { 391 Log.d(TAG, "handle() called: [" + data + "]"); 392 mDecoder.decodeAndApply(data); 393 } 394 395 static class SuggestionItem { 396 final InlineContentView mView; 397 final boolean mIsPinned; 398 SuggestionItem(InlineContentView view, boolean isPinned)399 SuggestionItem(InlineContentView view, boolean isPinned) { 400 mView = view; 401 mIsPinned = isPinned; 402 } 403 } 404 405 enum ResponseState { 406 RESET, 407 RECEIVE_RESPONSE, 408 START_INPUT, 409 } 410 } 411