freerdp: optional camera redirection

app/src/main/AndroidManifest.xml

@@ -25,6 +25,8 @@
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
     <uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+    <uses-permission android:name="android.permission.CAMERA" />
 
     <!--
         The lock screen in front of an export that carries the saved passwords

app/src/main/java/net/pgaskin/remotedesktop/SessionActivity.java

@@ -31,6 +31,7 @@
 import net.pgaskin.remotedesktop.control.input.TapRegions;
 import net.pgaskin.remotedesktop.session.SessionView;
 
+import java.util.HashMap;
 import java.util.Map;
 
 /**
@@ -161,6 +162,14 @@ protected void onCreate(Bundle savedInstanceState) {
      * are here as well.
      */
     private void start(String address, String backendId) {
+        final Map<String, String> wanted = connectionOptions();
+        final Runnable again = () -> start(address, backendId);
+        if (awaitingPermission(wanted, MICROPHONE_OPTION,
+                android.Manifest.permission.RECORD_AUDIO, REQUEST_MICROPHONE, again)
+                || awaitingPermission(wanted, CAMERA_OPTION,
+                android.Manifest.permission.CAMERA, REQUEST_CAMERA, again)) {
+            return;
+        }
         try {
             session = openOrAttach(address);
         } catch (Throwable t) {
@@ -359,6 +368,29 @@ private String sessionKey(String address) {
                 : Sessions.keyForAddress(address);
     }
 
+    /** What the connection asks for, as saved or as given on the command line. */
+    private Map<String, String> connectionOptions() {
+        return connection != null ? Options.forConnection(this, connection) : optionsFromIntent();
+    }
+
+    /**
+     * What the backend is built with: the requested options, except that a
+     * microphone or camera the phone has not been allowed to use is off rather
+     * than requested. A refused device is a session without it, not no session.
+     */
+    private Map<String, String> sessionOptions() {
+        final Map<String, String> options = connectionOptions();
+        final Map<String, String> without = new HashMap<>(options);
+        if (wanted(options, MICROPHONE_OPTION)
+                && !granted(android.Manifest.permission.RECORD_AUDIO)) {
+            without.put(MICROPHONE_OPTION, "false");
+        }
+        if (wanted(options, CAMERA_OPTION) && !granted(android.Manifest.permission.CAMERA)) {
+            without.put(CAMERA_OPTION, "false");
+        }
+        return without;
+    }
+
     private Session openOrAttach(String address) {
         final Session live = Sessions.byKey(sessionKey(address));
         if (live != null && !live.isClosed()) {
@@ -367,8 +399,7 @@ private Session openOrAttach(String address) {
         // A saved connection carries its own options, layered over its backend's;
         // an address from the command line gets the defaults. The application
         // context, because the backend outlives this activity by design.
-        final Map<String, String> options = connection != null
-                ? Options.forConnection(this, connection) : optionsFromIntent();
+        final Map<String, String> options = sessionOptions();
         final Backend backend = Backends.create(getApplicationContext(), backendId(), address,
                 connection != null ? connection.userName() : getIntent().getStringExtra("user"),
                 connection != null ? connection.password()
@@ -408,6 +439,59 @@ private void askAboutNotifications() {
         }
     }
 
+    /*
+     * The backend options that send the microphone and the camera, read by name
+     * as FreeRDP's provider names them. Each needs a permission from Android, and
+     * the connection is opened as soon as the session starts, with the device, so
+     * a connection that wants one waits for the answer.
+     */
+    private static final String MICROPHONE_OPTION = "Microphone";
+    private static final String CAMERA_OPTION = "Camera";
+    private static final int REQUEST_MICROPHONE = 2;
+    private static final int REQUEST_CAMERA = 3;
+
+    /** Asked once per window, so a refusal does not come back on every connect. */
+    private final java.util.Set<Integer> askedForPermission = new java.util.HashSet<>();
+    /** What to do once the permission question has been answered. */
+    private Runnable afterPermission;
+
+    private boolean wanted(Map<String, String> options, String option) {
+        return "true".equals(options.get(option));
+    }
+
+    private boolean granted(String permission) {
+        return checkSelfPermission(permission)
+                == android.content.pm.PackageManager.PERMISSION_GRANTED;
+    }
+
+    /**
+     * Returns true when the window is waiting for an answer to a permission
+     * this connection needs, and then {@code then} runs once it is in, whichever
+     * it was. A refusal is not an error: the session starts without the device.
+     */
+    private boolean awaitingPermission(Map<String, String> options, String option,
+                                       String permission, int request, Runnable then) {
+        if (!wanted(options, option) || granted(permission)
+                || !askedForPermission.add(request)) {
+            return false;
+        }
+        afterPermission = then;
+        requestPermissions(new String[]{permission}, request);
+        return true;
+    }
+
+    @Override
+    public void onRequestPermissionsResult(int requestCode, String[] permissions,
+                                           int[] grantResults) {
+        super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+        if ((requestCode == REQUEST_MICROPHONE || requestCode == REQUEST_CAMERA)
+                && afterPermission != null) {
+            final Runnable then = afterPermission;
+            afterPermission = null;
+            then.run();
+        }
+    }
+
     /**
      * Focus is told to the backend on the way <em>up</em> only, and from here
      * rather than from {@link #onStart}, because of the clipboard: the core

backends/freerdp/src/main/cpp/freerdp_jni.c

@@ -190,6 +190,10 @@ struct Session {
     char *experience;
     char *graphics;
     char *sound;
+    int microphone;
+    int camera;
+    char *gatewayHost; /* NULL for a direct connection */
+    int gatewayPort;
     char *clientName;
     char *configPath;
     int width, height, monitors, keyboardLayout;
@@ -953,19 +957,29 @@ static BOOL onAuthenticate(freerdp *instance, char **username, char **password,
 
    1 rather than 2 deliberately: 2 is "remember this", which would put the
    decision in a file of FreeRDP's beside the one this app already keeps. */
-static DWORD askTrust(Session *s, const char *fingerprint) {
+static DWORD askTrust(Session *s, const char *fingerprint, const char *host, UINT16 port) {
     pthread_mutex_lock(&s->lock);
     s->trustAnswered = 0;
     s->trustAccepted = 0;
     pthread_mutex_unlock(&s->lock);
 
+    /* The library asks for the gateway's certificate with the gateway's name, so
+       a host that is the gateway is the gateway's question. Java keeps its pin
+       under a different key, and the target's pins stay where they were. */
+    const int viaGateway = s->gatewayHost && host && strcmp(host, s->gatewayHost) == 0;
+    char hostPort[300];
+    snprintf(hostPort, sizeof(hostPort), "%s:%u", host ? host : "", (unsigned) port);
+
     int attached;
     JNIEnv *env = attach(s, &attached);
     if (env) {
         jstring js = (*env)->NewStringUTF(env, fingerprint ? fingerprint : "");
-        (*env)->CallVoidMethod(env, s->callbacks, mTrustNeeded, js);
+        jstring jhp = (*env)->NewStringUTF(env, hostPort);
+        (*env)->CallVoidMethod(env, s->callbacks, mTrustNeeded, js,
+                               viaGateway ? JNI_TRUE : JNI_FALSE, jhp);
         cleared(env);
         (*env)->DeleteLocalRef(env, js);
+        (*env)->DeleteLocalRef(env, jhp);
         detach(s, attached);
     }
 
@@ -981,8 +995,8 @@ static DWORD askTrust(Session *s, const char *fingerprint) {
 static DWORD onVerifyCertificate(freerdp *instance, const char *host, UINT16 port,
                                  const char *common_name, const char *subject, const char *issuer,
                                  const char *fingerprint, DWORD flags) {
-    (void) host; (void) port; (void) common_name; (void) subject; (void) issuer; (void) flags;
-    return askTrust(sessionOf(instance->context), fingerprint);
+    (void) common_name; (void) subject; (void) issuer; (void) flags;
+    return askTrust(sessionOf(instance->context), fingerprint, host, port);
 }
 
 /* Not the same question, and leaving it unset is a refusal: this is the one
@@ -993,9 +1007,9 @@ static DWORD onVerifyChangedCertificate(freerdp *instance, const char *host, UIN
                                         const char *issuer, const char *fingerprint,
                                         const char *old_subject, const char *old_issuer,
                                         const char *old_fingerprint, DWORD flags) {
-    (void) host; (void) port; (void) common_name; (void) subject; (void) issuer;
+    (void) common_name; (void) subject; (void) issuer;
     (void) old_subject; (void) old_issuer; (void) old_fingerprint; (void) flags;
-    return askTrust(sessionOf(instance->context), fingerprint);
+    return askTrust(sessionOf(instance->context), fingerprint, host, port);
 }
 
 /* ---- keysyms to scancodes ----------------------------------------------- */
@@ -1490,6 +1504,32 @@ static void onPostDisconnect(freerdp *instance) {
     gdi_free(instance);
 }
 
+/* "host" or "host:port", with 443 when there is no port. Empty is a direct
+   connection. IPv6 literals are not taken: a colon in the host is the port. */
+static void parseGateway(Session *s, const char *text) {
+    s->gatewayHost = NULL;
+    s->gatewayPort = 443;
+    if (!text || !*text) {
+        return;
+    }
+    size_t hostLen = strlen(text);
+    const char *colon = strrchr(text, ':');
+    char *end = NULL;
+    const long port = colon ? strtol(colon + 1, &end, 10) : 0;
+    if (colon && end && end != colon + 1 && *end == '\0' && port > 0 && port <= 65535) {
+        hostLen = (size_t) (colon - text);
+        s->gatewayPort = (int) port;
+    }
+    if (hostLen == 0) {
+        return;
+    }
+    s->gatewayHost = malloc(hostLen + 1);
+    if (s->gatewayHost) {
+        memcpy(s->gatewayHost, text, hostLen);
+        s->gatewayHost[hostLen] = '\0';
+    }
+}
+
 static void applySettings(Session *s, rdpSettings *settings) {
     freerdp_settings_set_string(settings, FreeRDP_ServerHostname, s->host);
     freerdp_settings_set_uint32(settings, FreeRDP_ServerPort, (UINT32) s->port);
@@ -1600,6 +1640,34 @@ static void applySettings(Session *s, rdpSettings *settings) {
     const int soundThere = s->sound && strcmp(s->sound, "remote") == 0;
     freerdp_settings_set_bool(settings, FreeRDP_AudioPlayback, soundHere);
     freerdp_settings_set_bool(settings, FreeRDP_RemoteConsoleAudio, soundThere);
+    /* The microphone is the audin channel's capture side, which the library
+       opens through OpenSL ES. The app checks RECORD_AUDIO before it gets here,
+       since the capture device cannot be opened without it. */
+    freerdp_settings_set_bool(settings, FreeRDP_AudioCapture, s->microphone);
+
+    /* The camera is the rdpecam dynamic channel, which the library adds by name
+       the same way its own client does with /dvc:rdpecam. Its Android backend
+       is the Camera2 NDK, and the app has checked CAMERA before it gets here. */
+    if (s->camera) {
+        static const char *const cameraChannel[] = {"rdpecam"};
+        if (!freerdp_client_add_dynamic_channel(settings, 1, cameraChannel)) {
+            LOGW("could not add the camera channel");
+        }
+    }
+
+    /* RD Gateway, over HTTPS only (the RPC transport is the old one). The gateway
+       takes the connection's own credentials, so there is nothing separate to
+       store. TSC_PROXY_MODE_DIRECT means always use it, where the detect mode
+       would only use it when the direct route fails. */
+    freerdp_settings_set_bool(settings, FreeRDP_GatewayEnabled, s->gatewayHost != NULL);
+    if (s->gatewayHost) {
+        freerdp_settings_set_string(settings, FreeRDP_GatewayHostname, s->gatewayHost);
+        freerdp_settings_set_uint32(settings, FreeRDP_GatewayPort, (UINT32) s->gatewayPort);
+        freerdp_settings_set_uint32(settings, FreeRDP_GatewayUsageMethod, TSC_PROXY_MODE_DIRECT);
+        freerdp_settings_set_bool(settings, FreeRDP_GatewayUseSameCredentials, TRUE);
+        freerdp_settings_set_bool(settings, FreeRDP_GatewayHttpTransport, TRUE);
+        freerdp_settings_set_bool(settings, FreeRDP_GatewayRpcTransport, FALSE);
+    }
 
     /* How big the far end draws its own interface. The protocol allows 100–500
        for the desktop factor and exactly 100, 140 or 180 for the device one,
@@ -1785,7 +1853,9 @@ JNIEXPORT jlong JNICALL
 Java_net_pgaskin_remotedesktop_backend_freerdp_FreeRdpNative_nativeCreate(
         JNIEnv *env, jclass cls, jobject listener, jstring address, jstring userName,
         jstring domain, jstring password, jstring nla, jboolean legacyTls, jboolean compression,
-        jstring graphics, jstring experience, jstring sound, jint scale, jint width, jint height,
+        jstring graphics, jstring experience, jstring sound, jboolean microphone,
+        jboolean camera, jstring gateway, jint scale,
+        jint width, jint height,
         jint monitors, jint keyboardLayout, jstring clientName, jstring configPath,
         jint connectTimeoutMs) {
     (void) cls;
@@ -1799,7 +1869,8 @@ Java_net_pgaskin_remotedesktop_backend_freerdp_FreeRdpNative_nativeCreate(
         mBell = (*env)->GetMethodID(env, k, "onBell", "()V");
         mClipboard = (*env)->GetMethodID(env, k, "onClipboard", "(Ljava/lang/String;)V");
         mCredentialsNeeded = (*env)->GetMethodID(env, k, "onCredentialsNeeded", "(Z)V");
-        mTrustNeeded = (*env)->GetMethodID(env, k, "onTrustNeeded", "(Ljava/lang/String;)V");
+        mTrustNeeded = (*env)->GetMethodID(env, k, "onTrustNeeded",
+                                           "(Ljava/lang/String;ZLjava/lang/String;)V");
         mClosed = (*env)->GetMethodID(env, k, "onClosed", "(Ljava/lang/String;)V");
         /* Last, and that is the point: it is what every other caller tests, so
            publishing it first would let a second one past while the method ids
@@ -1836,6 +1907,11 @@ Java_net_pgaskin_remotedesktop_backend_freerdp_FreeRdpNative_nativeCreate(
     s->graphics = dup_jstring(env, graphics);
     s->experience = dup_jstring(env, experience);
     s->sound = dup_jstring(env, sound);
+    s->microphone = microphone != JNI_FALSE;
+    s->camera = camera != JNI_FALSE;
+    char *gatewayText = dup_jstring(env, gateway);
+    parseGateway(s, gatewayText);
+    free(gatewayText);
     s->clientName = dup_jstring(env, clientName);
     s->configPath = dup_jstring(env, configPath);
     s->compression = compression != JNI_FALSE;
@@ -1896,6 +1972,7 @@ Java_net_pgaskin_remotedesktop_backend_freerdp_FreeRdpNative_nativeCreate(
     free(s->graphics);
     free(s->experience);
     free(s->sound);
+    free(s->gatewayHost);
     free(s->clientName);
     free(s->configPath);
     free(s);
@@ -2008,6 +2085,7 @@ Java_net_pgaskin_remotedesktop_backend_freerdp_FreeRdpNative_nativeDestroy(
     free(s->graphics);
     free(s->experience);
     free(s->sound);
+    free(s->gatewayHost);
     free(s->clientName);
     free(s->configPath);
     free(s->credUser);

backends/freerdp/src/main/java/net/pgaskin/remotedesktop/backend/freerdp/FreeRdpBackend.java

@@ -184,7 +184,9 @@ public void connect() {
                 options.get(FreeRdpProvider.NLA), bool(FreeRdpProvider.LEGACY_TLS),
                 bool(FreeRdpProvider.COMPRESSION),
                 options.get(FreeRdpProvider.GRAPHICS), options.get(FreeRdpProvider.EXPERIENCE),
-                options.get(FreeRdpProvider.SOUND), scale(),
+                options.get(FreeRdpProvider.SOUND), bool(FreeRdpProvider.MICROPHONE),
+                bool(FreeRdpProvider.CAMERA),
+                options.getOrDefault(FreeRdpProvider.GATEWAY, ""), scale(),
                 size[0], size[1], monitorCount(), KEYBOARD_LAYOUT,
                 android.os.Build.MODEL, store.getAbsolutePath(), CONNECT_TIMEOUT_MS);
         if (h == 0) {
@@ -540,13 +542,16 @@ protected void deliver(boolean ok, String user, String secret) {
      * what is identified is the machine, not what reaches it.
      */
     @Override
-    public void onTrustNeeded(String fingerprint) {
-        if (KnownHosts.matches(context, address, fingerprint)) {
-            Log.i(TAG, "certificate matches the pin for " + address);
+    public void onTrustNeeded(String fingerprint, boolean gateway, String hostPort) {
+        // The gateway is a different machine with its own certificate, so it is
+        // pinned under its own name; the target keeps the address it was saved under.
+        final String key = gateway ? "RD Gateway " + hostPort : address;
+        if (KnownHosts.matches(context, key, fingerprint)) {
+            Log.i(TAG, "certificate matches the pin for " + key);
             answerTrust(true);
             return;
         }
-        main.post(() -> KnownHosts.ask(context, address, fingerprint, prompts, this::answerTrust));
+        main.post(() -> KnownHosts.ask(context, key, fingerprint, prompts, this::answerTrust));
     }
 
     private void answerTrust(boolean accept) {

backends/freerdp/src/main/java/net/pgaskin/remotedesktop/backend/freerdp/FreeRdpNative.java

@@ -82,8 +82,10 @@ interface Callbacks {
          * secret is sent.
          *
          * @param fingerprint SHA-256, as {@code openssl x509 -fingerprint} prints it
+         * @param gateway    the certificate is the RD Gateway's, not the machine's
+         * @param hostPort   the host and port the certificate was presented for
          */
-        void onTrustNeeded(String fingerprint);
+        void onTrustNeeded(String fingerprint, boolean gateway, String hostPort);
 
         /** Terminal. Empty {@code detail} for an ordinary disconnect. */
         void onClosed(String detail);
@@ -104,6 +106,11 @@ interface Callbacks {
      *                   {@code bitmap}
      * @param experience {@code full}, {@code balanced} or {@code plain}
      * @param sound      {@code off}, {@code local} or {@code remote}
+     * @param microphone send this phone's microphone; the caller only passes
+     *                   true once RECORD_AUDIO is granted
+     * @param camera     send this phone's camera; the caller only passes true
+     *                   once CAMERA is granted
+     * @param gateway    an RD Gateway as host or host:port; empty for none
      * @param scale      the percentage the far end is asked to draw its own
      *                   interface at; 100, 140 and 180 are the only values the
      *                   protocol allows, and anything else is taken as 100
@@ -114,7 +121,8 @@ interface Callbacks {
     static native long nativeCreate(Callbacks listener, String address, String userName,
                                     String domain, String password, String nla,
                                     boolean legacyTls, boolean compression, String graphics,
-                                    String experience, String sound, int scale, int width,
+                                    String experience, String sound, boolean microphone,
+                                    boolean camera, String gateway, int scale, int width,
                                     int height, int monitors, int keyboardLayout,
                                     String clientName, String configPath, int connectTimeoutMs);