Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Add support for image insertion on Android #35619

Merged
merged 10 commits into from
Oct 20, 2022
Merged

Add support for image insertion on Android #35619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
4a68d3d
Add support for image insertion on Android
tneotia Aug 23, 2022
ebbf8d5
Fix formatting and compile errors
tneotia Aug 23, 2022
6b3793b
Catch IOException
tneotia Aug 23, 2022
eee16ef
Fix formatting and unrecognized symbol
tneotia Aug 23, 2022
0fa0edd
Fix nits and release permission in every case
tneotia Sep 14, 2022
6313ccb
Merge remote-tracking branch 'upstream/master' into gif-insertion
tneotia Sep 15, 2022
d6d7031
Fix test
justinmc Oct 18, 2022
db0cfee
Merge remote-tracking branch 'upstream/master' into gif-insertion
tneotia Oct 19, 2022
76e8c2a
Merge remote-tracking branch 'upstream/master' into gif-insertion
tneotia Oct 19, 2022
97a2d01
Make method private
tneotia Oct 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions shell/platform/android/io/flutter/embedding/engine/systemchannels/TextInputChannel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.json.JSONArray;
Expand Down Expand Up @@ -325,6 +326,14 @@ public void unspecifiedAction(int inputClientId) {
Arrays.asList(inputClientId, "TextInputAction.unspecified"));
}

/** Instructs Flutter to commit inserted content back to the text channel. */
public void commitContent(int inputClientId, Map<String, Object> content) {
Log.v(TAG, "Sending 'commitContent' message.");
channel.invokeMethod(
"TextInputClient.performAction",
Arrays.asList(inputClientId, "TextInputAction.commitContent", content));
}

public void performPrivateCommand(
int inputClientId, @NonNull String action, @NonNull Bundle data) {
HashMap<Object, Object> json = new HashMap<>();
Expand Down Expand Up @@ -454,6 +463,19 @@ public static Configuration fromJson(@NonNull JSONObject json)
}
}
final Integer inputAction = inputActionFromTextInputAction(inputActionName);

// Build list of content commit mime types from the data in the JSON list.
List<String> contentList = new ArrayList<String>();
JSONArray contentCommitMimeTypes =
json.isNull("contentCommitMimeTypes")
? null
: json.getJSONArray("contentCommitMimeTypes");
if (contentCommitMimeTypes != null) {
for (int i = 0; i < contentCommitMimeTypes.length(); i++) {
contentList.add(contentCommitMimeTypes.optString(i));
}
}

return new Configuration(
json.optBoolean("obscureText"),
json.optBoolean("autocorrect", true),
Expand All @@ -465,6 +487,7 @@ public static Configuration fromJson(@NonNull JSONObject json)
inputAction,
json.isNull("actionLabel") ? null : json.getString("actionLabel"),
json.isNull("autofill") ? null : Autofill.fromJson(json.getJSONObject("autofill")),
contentList.toArray(new String[contentList.size()]),
fields);
}

Expand Down Expand Up @@ -622,6 +645,7 @@ public Autofill(
@Nullable public final Integer inputAction;
@Nullable public final String actionLabel;
@Nullable public final Autofill autofill;
@Nullable public final String[] contentCommitMimeTypes;
@Nullable public final Configuration[] fields;

public Configuration(
Expand All @@ -635,6 +659,7 @@ public Configuration(
@Nullable Integer inputAction,
@Nullable String actionLabel,
@Nullable Autofill autofill,
@Nullable String[] contentCommitMimeTypes,
@Nullable Configuration[] fields) {
this.obscureText = obscureText;
this.autocorrect = autocorrect;
Expand All @@ -646,6 +671,7 @@ public Configuration(
this.inputAction = inputAction;
this.actionLabel = actionLabel;
this.autofill = autofill;
this.contentCommitMimeTypes = contentCommitMimeTypes;
this.fields = fields;
}
}
Expand Down
80 changes: 80 additions & 0 deletions shell/platform/android/io/flutter/plugin/editing/InputConnectionAdaptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@

package io.flutter.plugin.editing;

import android.annotation.TargetApi;
import android.content.ClipData;
import android.content.ClipboardManager;
import android.content.Context;
import android.net.Uri;
import android.os.Build;
import android.os.Bundle;
import android.text.DynamicLayout;
Expand All @@ -22,11 +24,20 @@
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.ExtractedText;
import android.view.inputmethod.ExtractedTextRequest;
import android.view.inputmethod.InputContentInfo;
import android.view.inputmethod.InputMethodManager;
import androidx.annotation.NonNull;
import androidx.annotation.RequiresApi;
import androidx.core.view.inputmethod.InputConnectionCompat;
import io.flutter.Log;
import io.flutter.embedding.engine.FlutterJNI;
import io.flutter.embedding.engine.systemchannels.TextInputChannel;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;

public class InputConnectionAdaptor extends BaseInputConnection
implements ListenableEditingState.EditingStateWatcher {
Expand Down Expand Up @@ -477,6 +488,75 @@ public boolean performEditorAction(int actionCode) {
return true;
}

@Override
@TargetApi(25)
@RequiresApi(25)
public boolean commitContent(InputContentInfo inputContentInfo, int flags, Bundle opts) {
// Ensure permission is granted.
if (Build.VERSION.SDK_INT >= 25
&& (flags & InputConnectionCompat.INPUT_CONTENT_GRANT_READ_URI_PERMISSION) != 0) {
try {
inputContentInfo.requestPermission();
} catch (Exception e) {
return false;
}
} else {
return false;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What will this end up looking like in the framework if this isn't supported by the device for some reason? Ideally an app developer would be able to gracefully handle this situation.

Copy link
Contributor Author

@tneotia tneotia Sep 14, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's a toast message displayed that "<app> doesn't support image insertion here". It will display this toast any time this function returns false. To my knowledge, it cannot be customized and is an Android framework thing.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok sounds like that's the same as native so that should be fine 👍

}

if (inputContentInfo.getDescription().getMimeTypeCount() > 0) {
inputContentInfo.requestPermission();

final Uri uri = inputContentInfo.getContentUri();
final String mimeType = inputContentInfo.getDescription().getMimeType(0);
Context context = mFlutterView.getContext();

if (uri != null) {
InputStream is;
try {
// Extract byte data from the given URI.
is = context.getContentResolver().openInputStream(uri);
} catch (FileNotFoundException ex) {
inputContentInfo.releasePermission();
return false;
}

if (is != null) {
final byte[] data = this.readStreamFully(is, 64 * 1024);

final Map<String, Object> obj = new HashMap<>();
obj.put("mimeType", mimeType);
obj.put("data", data);
obj.put("uri", uri.toString());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are all 3 of these parameters guaranteed to be valid at this point, or should there be any checking and error handling here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure to be honest. This is why we made the CommittedContent class parameters nullable, just in case something doesn't work correctly.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So I guess it will be up to the framework to handle an invalid CommittedContent.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct, or the developer themselves. They can choose what they want to do if a parameter they need happens to be null.


// Commit the content to the text input channel and release the permission.
textInputChannel.commitContent(mClient, obj);
inputContentInfo.releasePermission();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you have to release permission when is is null? You request it in line 508 but only release when successful here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed. Good catch :)

return true;
}
}

inputContentInfo.releasePermission();
}
return false;
}

private byte[] readStreamFully(InputStream is, int blocksize) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();

byte[] buffer = new byte[blocksize];
while (true) {
int len = -1;
try {
len = is.read(buffer);
} catch (IOException ex) {
}
if (len == -1) break;
baos.write(buffer, 0, len);
}
return baos.toByteArray();
}

// -------- Start: ListenableEditingState watcher implementation -------
@Override
public void didChangeEditingState(
Expand Down
6 changes: 6 additions & 0 deletions shell/platform/android/io/flutter/plugin/editing/TextInputPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.core.view.inputmethod.EditorInfoCompat;
import io.flutter.Log;
import io.flutter.embedding.android.KeyboardManager;
import io.flutter.embedding.engine.systemchannels.TextInputChannel;
Expand Down Expand Up @@ -342,6 +343,11 @@ public InputConnection createInputConnection(
}
outAttrs.imeOptions |= enterAction;

if (configuration.contentCommitMimeTypes != null) {
String[] imgTypeString = configuration.contentCommitMimeTypes;
EditorInfoCompat.setContentMimeTypes(outAttrs, imgTypeString);
}

InputConnectionAdaptor connection =
new InputConnectionAdaptor(
view, inputTarget.id, textInputChannel, keyboardManager, mEditable, outAttrs);
Expand Down
73 changes: 73 additions & 0 deletions shell/platform/android/test/io/flutter/plugin/editing/InputConnectionAdaptorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,12 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import android.content.ClipDescription;
import android.content.ClipboardManager;
import android.content.ContentResolver;
import android.content.Context;
import android.content.res.AssetManager;
import android.net.Uri;
import android.os.Build;
import android.os.Bundle;
import android.text.InputType;
Expand All @@ -31,7 +34,9 @@
import android.view.inputmethod.ExtractedText;
import android.view.inputmethod.ExtractedTextRequest;
import android.view.inputmethod.InputConnection;
import android.view.inputmethod.InputContentInfo;
import android.view.inputmethod.InputMethodManager;
import androidx.core.view.inputmethod.InputConnectionCompat;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.ibm.icu.lang.UCharacter;
Expand All @@ -43,7 +48,9 @@
import io.flutter.plugin.common.JSONMethodCodec;
import io.flutter.plugin.common.MethodCall;
import io.flutter.util.FakeKeyEvent;
import java.io.ByteArrayInputStream;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import org.json.JSONArray;
import org.json.JSONException;
import org.junit.Before;
Expand All @@ -52,10 +59,12 @@
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.robolectric.Shadows;
import org.robolectric.annotation.Config;
import org.robolectric.annotation.Implementation;
import org.robolectric.annotation.Implements;
import org.robolectric.shadow.api.Shadow;
import org.robolectric.shadows.ShadowContentResolver;
import org.robolectric.shadows.ShadowInputMethodManager;

@Config(
Expand All @@ -64,6 +73,9 @@
@RunWith(AndroidJUnit4.class)
public class InputConnectionAdaptorTest {
private final Context ctx = ApplicationProvider.getApplicationContext();
private ContentResolver contentResolver;
private ShadowContentResolver shadowContentResolver;

@Mock KeyboardManager mockKeyboardManager;
// Verifies the method and arguments for a captured method call.
private void verifyMethodCall(ByteBuffer buffer, String methodName, String[] expectedArgs)
Expand All @@ -83,6 +95,8 @@ private void verifyMethodCall(ByteBuffer buffer, String methodName, String[] exp
@Before
public void setUp() {
MockitoAnnotations.openMocks(this);
contentResolver = ctx.getContentResolver();
shadowContentResolver = Shadows.shadowOf(contentResolver);
}

@Test
Expand Down Expand Up @@ -171,6 +185,65 @@ public void testPerformContextMenuAction_paste() {
assertTrue(editable.toString().startsWith(textToBePasted));
}

@Test
public void testCommitContent() throws JSONException {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Heads up that this test appears to be failing


io.flutter.plugin.editing.InputConnectionAdaptorTest > testCommitContent FAILED
    Wanted but not invoked:
    dartExecutor.send(
        <Capturing argument>,
        <Capturing argument>,
        isNull()
    );
    -> at io.flutter.embedding.engine.dart.DartExecutor.send(DartExecutor.java:223)

    However, there was exactly 1 interaction with this mock:
    dartExecutor.setMessageHandler(
        "flutter/textinput",
        io.flutter.plugin.common.MethodChannel$IncomingMethodCallHandler@18698c6a
    );
    -> at io.flutter.plugin.common.MethodChannel.setMethodCallHandler(MethodChannel.java:146)
        at io.flutter.embedding.engine.dart.DartExecutor.send(DartExecutor.java:223)
        at io.flutter.plugin.editing.InputConnectionAdaptorTest.testCommitContent(InputConnectionAdaptorTest.java:203)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm aware of this, the error hasn't really helped me to figure out why its failing. No idea what needs to change to get it to work.

View testView = new View(ctx);
int client = 0;
FlutterJNI mockFlutterJNI = mock(FlutterJNI.class);
DartExecutor dartExecutor = spy(new DartExecutor(mockFlutterJNI, mock(AssetManager.class)));
TextInputChannel textInputChannel = new TextInputChannel(dartExecutor);
ListenableEditingState editable = sampleEditable(0, 0);
InputConnectionAdaptor adaptor =
new InputConnectionAdaptor(
testView,
client,
textInputChannel,
mockKeyboardManager,
editable,
null,
mockFlutterJNI);

String uri = "content://mock/uri/test/commitContent";
Charset charset = Charset.forName("UTF-8");
String fakeImageData = "fake image data";
byte[] fakeImageDataBytes = fakeImageData.getBytes(charset);
shadowContentResolver.registerInputStream(
Uri.parse(uri), new ByteArrayInputStream(fakeImageDataBytes));

boolean commitContentSuccess =
adaptor.commitContent(
new InputContentInfo(
Uri.parse(uri),
new ClipDescription("commitContent test", new String[] {"image/png"})),
InputConnectionCompat.INPUT_CONTENT_GRANT_READ_URI_PERMISSION,
null);
assertTrue(commitContentSuccess);

ArgumentCaptor<String> channelCaptor = ArgumentCaptor.forClass(String.class);
ArgumentCaptor<ByteBuffer> bufferCaptor = ArgumentCaptor.forClass(ByteBuffer.class);
verify(dartExecutor, times(1)).send(channelCaptor.capture(), bufferCaptor.capture(), isNull());
assertEquals("flutter/textinput", channelCaptor.getValue());

String fakeImageDataIntString = "";
for (int i = 0; i < fakeImageDataBytes.length; i++) {
int byteAsInt = fakeImageDataBytes[i];
fakeImageDataIntString += byteAsInt;
if (i < (fakeImageDataBytes.length - 1)) {
fakeImageDataIntString += ",";
}
}
verifyMethodCall(
bufferCaptor.getValue(),
"TextInputClient.performAction",
new String[] {
"0",
"TextInputAction.commitContent",
"{\"data\":["
+ fakeImageDataIntString
+ "],\"mimeType\":\"image\\/png\",\"uri\":\"content:\\/\\/mock\\/uri\\/test\\/commitContent\"}"
});
}

@Test
public void testPerformPrivateCommand_dataIsNull() throws JSONException {
View testView = new View(ctx);
Expand Down
Loading