~singpolyma/cheogram-android

4d26b28307fef0b9d2fce37c9b08f790afee64a8 — Stephen Paul Weber 4 months ago 4e42570
Support urn:ietf:rfc:3264

The Jingle ICE-UDP spec says that if the other side advertises this
feature then we should put our candidates into session-initiate or
session-accept instead of trickle-ing them out one at a time as per usual.
M src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java => src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +87 -41
@@ 26,6 26,7 @@ import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;

import org.webrtc.DtmfSender;
import org.webrtc.EglBase;


@@ 184,6 185,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
    private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();
    private ScheduledFuture<?> ringingTimeoutFuture;
    private final long created = System.currentTimeMillis() / 1000L;
    private final SettableFuture<Collection<IceCandidate>> iceGatheringComplete = SettableFuture.create();

    JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) {
        super(jingleConnectionManager, id, initiator);


@@ 1127,6 1129,8 @@ public class JingleRtpConnection extends AbstractJingleConnection
            }
            return;
        }
        final Presence presence = id.getContact().getPresences().get(id.getWith().getResource());
        if (presence != null && presence.getServiceDiscoveryResult().getFeatures().contains("urn:ietf:rfc:3264")) webRTCWrapper.setRFC3264(true);
        final ListenableFuture<RtpContentMap> future = receiveRtpContentMap(jinglePacket, false);
        Futures.addCallback(
                future,


@@ 1395,29 1399,46 @@ public class JingleRtpConnection extends AbstractJingleConnection
    }

    private void prepareSessionAccept(
            final org.webrtc.SessionDescription webRTCSessionDescription) {
        final SessionDescription sessionDescription =
                SessionDescription.parse(webRTCSessionDescription.description);
        final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
        this.responderRtpContentMap = respondingRtpContentMap;
        storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
        final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
                prepareOutgoingContentMap(respondingRtpContentMap);
            final org.webrtc.SessionDescription initialWebRTCSessionDescription) {
        Futures.addCallback(
                outgoingContentMapFuture,
                new FutureCallback<RtpContentMap>() {
                    @Override
                    public void onSuccess(final RtpContentMap outgoingContentMap) {
                        sendSessionAccept(outgoingContentMap);
                        webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
                    }
            webRTCWrapper.getRFC3264() ? Futures.withTimeout(iceGatheringComplete, 2, TimeUnit.SECONDS, JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE) : Futures.immediateFuture(null),
            new FutureCallback<Collection<IceCandidate>>() {
                @Override
                public void onSuccess(final Collection<IceCandidate> iceCandidates) {
                    org.webrtc.SessionDescription webRTCSessionDescription =
                        JingleRtpConnection.this.webRTCWrapper.getLocalDescription();
                    final SessionDescription sessionDescription =
                        SessionDescription.parse(webRTCSessionDescription.description);
                    final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false);
                    JingleRtpConnection.this.responderRtpContentMap = respondingRtpContentMap;
                    storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip());
                    final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
                        prepareOutgoingContentMap(respondingRtpContentMap);
                    Futures.addCallback(
                        outgoingContentMapFuture,
                        new FutureCallback<RtpContentMap>() {
                            @Override
                            public void onSuccess(final RtpContentMap outgoingContentMap) {
                                sendSessionAccept(outgoingContentMap);
                                webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
                            }

                    @Override
                    public void onFailure(@NonNull Throwable throwable) {
                        failureToAcceptSession(throwable);
                    }
                },
                MoreExecutors.directExecutor());
                            @Override
                            public void onFailure(@NonNull Throwable throwable) {
                                failureToAcceptSession(throwable);
                            }
                        },
                        MoreExecutors.directExecutor());
                }

                @Override
                public void onFailure(@NonNull final Throwable throwable) {
                    Log.e(Config.LOGTAG, "ICE gathering didn't finish clean: " + throwable);
                    onSuccess(null);
                }
            },
            MoreExecutors.directExecutor()
        );
    }

    private void sendSessionAccept(final RtpContentMap rtpContentMap) {


@@ 1848,28 1869,46 @@ public class JingleRtpConnection extends AbstractJingleConnection
    }

    private void prepareSessionInitiate(
            final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) {
        final SessionDescription sessionDescription =
                SessionDescription.parse(webRTCSessionDescription.description);
        final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
        this.initiatorRtpContentMap = rtpContentMap;
        final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
                encryptSessionInitiate(rtpContentMap);
            final org.webrtc.SessionDescription initialWebRTCSessionDescription, final State targetState) {
        Futures.addCallback(
                outgoingContentMapFuture,
                new FutureCallback<RtpContentMap>() {
                    @Override
                    public void onSuccess(final RtpContentMap outgoingContentMap) {
                        sendSessionInitiate(outgoingContentMap, targetState);
                        webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
                    }
            webRTCWrapper.getRFC3264() ? Futures.withTimeout(iceGatheringComplete, 2, TimeUnit.SECONDS, JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE) : Futures.immediateFuture(null),
            new FutureCallback<Collection<IceCandidate>>() {
                @Override
                public void onSuccess(final Collection<IceCandidate> iceCandidates) {
                    org.webrtc.SessionDescription webRTCSessionDescription =
                        JingleRtpConnection.this.webRTCWrapper.getLocalDescription();
                    final SessionDescription sessionDescription =
                        SessionDescription.parse(webRTCSessionDescription.description);
                    final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true);
                    JingleRtpConnection.this.initiatorRtpContentMap = rtpContentMap;
                    final ListenableFuture<RtpContentMap> outgoingContentMapFuture =
                        encryptSessionInitiate(rtpContentMap);
                    Futures.addCallback(
                        outgoingContentMapFuture,
                        new FutureCallback<RtpContentMap>() {
                            @Override
                            public void onSuccess(final RtpContentMap outgoingContentMap) {
                                sendSessionInitiate(outgoingContentMap, targetState);
                                webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
                            }

                    @Override
                    public void onFailure(@NonNull final Throwable throwable) {
                        failureToInitiateSession(throwable, targetState);
                    }
                },
                MoreExecutors.directExecutor());
                            @Override
                            public void onFailure(@NonNull final Throwable throwable) {
                                failureToInitiateSession(throwable, targetState);
                            }
                        },
                        MoreExecutors.directExecutor()
                    );
                }

                @Override
                public void onFailure(@NonNull final Throwable throwable) {
                    Log.e(Config.LOGTAG, "ICE gathering didn't finish clean: " + throwable);
                    onSuccess(null);
                }
            },
            MoreExecutors.directExecutor()
        );
    }

    private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) {


@@ 2336,6 2375,8 @@ public class JingleRtpConnection extends AbstractJingleConnection

    private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
        this.jingleConnectionManager.ensureConnectionIsRegistered(this);
        final Presence presence = id.getContact().getPresences().get(id.getWith().getResource());
        if (presence != null && presence.getServiceDiscoveryResult().getFeatures().contains("urn:ietf:rfc:3264")) webRTCWrapper.setRFC3264(true);
        this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
        this.webRTCWrapper.initializePeerConnection(media, iceServers);
    }


@@ 2459,6 2500,11 @@ public class JingleRtpConnection extends AbstractJingleConnection
    }

    @Override
    public void onIceGatheringComplete(Collection<IceCandidate> iceCandidates) {
        iceGatheringComplete.set(iceCandidates);
    }

    @Override
    public void onConnectionChange(final PeerConnection.PeerConnectionState newState) {
        Log.d(
                Config.LOGTAG,

M src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java => src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +31 -5
@@ 41,6 41,7 @@ import org.webrtc.SessionDescription;
import org.webrtc.VideoTrack;
import org.webrtc.audio.JavaAudioDeviceModule;

import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;


@@ 102,6 103,7 @@ public class WebRTCWrapper {

    private final EventCallback eventCallback;
    private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false);
    private final AtomicBoolean rfc3264 = new AtomicBoolean(false);
    private final Queue<IceCandidate> iceCandidates = new LinkedList<>();
    private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents =
            new AppRTCAudioManager.AudioManagerEvents() {


@@ 152,11 154,14 @@ public class WebRTCWrapper {
                public void onIceGatheringChange(
                        PeerConnection.IceGatheringState iceGatheringState) {
                    Log.d(EXTENDED_LOGGING_TAG, "onIceGatheringChange(" + iceGatheringState + ")");
                    if (iceGatheringState == PeerConnection.IceGatheringState.COMPLETE) {
                        eventCallback.onIceGatheringComplete(iceCandidates);
                    }
                }

                @Override
                public void onIceCandidate(IceCandidate iceCandidate) {
                    if (readyToReceivedIceCandidates.get()) {
                    if (readyToReceivedIceCandidates.get() && !rfc3264.get()) {
                        eventCallback.onIceCandidate(iceCandidate);
                    } else {
                        iceCandidates.add(iceCandidate);


@@ 305,7 310,7 @@ public class WebRTCWrapper {
                                        .createAudioDeviceModule())
                        .createPeerConnectionFactory();

        final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers);
        final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers, rfc3264.get());
        final PeerConnection peerConnection =
                requirePeerConnectionFactory()
                        .createPeerConnection(rtcConfig, peerConnectionObserver);


@@ 420,13 425,18 @@ public class WebRTCWrapper {
    }

    private static PeerConnection.RTCConfiguration buildConfiguration(
            final List<PeerConnection.IceServer> iceServers) {
            final List<PeerConnection.IceServer> iceServers, boolean rfc3264) {
        final PeerConnection.RTCConfiguration rtcConfig =
                new PeerConnection.RTCConfiguration(iceServers);
        rtcConfig.tcpCandidatePolicy =
                PeerConnection.TcpCandidatePolicy.DISABLED; // XEP-0176 doesn't support tcp
        rtcConfig.continualGatheringPolicy =
        if (rfc3264) {
            rtcConfig.continualGatheringPolicy =
                PeerConnection.ContinualGatheringPolicy.GATHER_ONCE;
        } else {
            rtcConfig.continualGatheringPolicy =
                PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY;
        }
        rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN;
        rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE;
        rtcConfig.enableImplicitRollback = true;


@@ 434,7 444,7 @@ public class WebRTCWrapper {
    }

    void reconfigurePeerConnection(final List<PeerConnection.IceServer> iceServers) {
        requirePeerConnection().setConfiguration(buildConfiguration(iceServers));
        requirePeerConnection().setConfiguration(buildConfiguration(iceServers, rfc3264.get()));
    }

    void restartIceAsync() {


@@ 455,6 465,7 @@ public class WebRTCWrapper {

    public void setIsReadyToReceiveIceCandidates(final boolean ready) {
        readyToReceivedIceCandidates.set(ready);
        if (this.rfc3264.get()) return;
        final int was = iceCandidates.size();
        while (ready && iceCandidates.peek() != null) {
            eventCallback.onIceCandidate(iceCandidates.poll());


@@ 465,6 476,15 @@ public class WebRTCWrapper {
                "setIsReadyToReceiveCandidates(" + ready + ") was=" + was + " is=" + is);
    }

    public void setRFC3264(final boolean rfc3264) {
        // When this feature is enabled, do not trickle candidates
        this.rfc3264.set(rfc3264);
    }

    public boolean getRFC3264() {
        return this.rfc3264.get();
    }

    synchronized void close() {
        final PeerConnection peerConnection = this.peerConnection;
        final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory;


@@ 595,6 615,10 @@ public class WebRTCWrapper {
        throw new IllegalStateException("Local video track does not exist");
    }

    synchronized SessionDescription getLocalDescription() {
        return peerConnection.getLocalDescription();
    }

    synchronized ListenableFuture<SessionDescription> setLocalDescription() {
        this.setIsReadyToReceiveIceCandidates(false);
        return Futures.transformAsync(


@@ 769,6 793,8 @@ public class WebRTCWrapper {
                Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);

        void onRenegotiationNeeded();

        void onIceGatheringComplete(Collection<IceCandidate> iceCandidates);
    }

    private abstract static class SetSdpObserver implements SdpObserver {

M src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java => src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +3 -0
@@ 69,6 69,9 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
        for (final String iceOption : IceOption.of(media)) {
            iceUdpTransportInfo.addChild(new IceOption(iceOption));
        }
        for (final String candidate : media.attributes.get("candidate")) {
            iceUdpTransportInfo.addChild(Candidate.fromSdpAttributeValue(candidate, ufrag));
        }
        return iceUdpTransportInfo;
    }