~singpolyma/cheogram-android

4e8ceadfbf9f38cc9a53b7e912c9a7eb926dc2df — Daniel Gultsch 1 year, 4 months ago 63501ad
prepare JingleRtpConnection for content-adds
M src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java => src/main/java/eu/siacs/conversations/services/AppRTCAudioManager.java +21 -2
@@ 33,6 33,7 @@ import java.util.concurrent.CountDownLatch;

import eu.siacs.conversations.Config;
import eu.siacs.conversations.utils.AppRTCUtils;
import eu.siacs.conversations.xmpp.jingle.Media;

/**
 * AppRTCAudioManager manages all audio related parts of the AppRTC demo.


@@ 44,7 45,7 @@ public class AppRTCAudioManager {
    private final Context apprtcContext;
    // Contains speakerphone setting: auto, true or false
    @Nullable
    private final SpeakerPhonePreference speakerPhonePreference;
    private SpeakerPhonePreference speakerPhonePreference;
    // Handles all tasks related to Bluetooth headset devices.
    private final AppRTCBluetoothManager bluetoothManager;
    @Nullable


@@ 110,6 111,16 @@ public class AppRTCAudioManager {
        AppRTCUtils.logDeviceInfo(Config.LOGTAG);
    }

    public void switchSpeakerPhonePreference(final SpeakerPhonePreference speakerPhonePreference) {
        this.speakerPhonePreference = speakerPhonePreference;
        if (speakerPhonePreference == SpeakerPhonePreference.EARPIECE && hasEarpiece()) {
            defaultAudioDevice = AudioDevice.EARPIECE;
        } else {
            defaultAudioDevice = AudioDevice.SPEAKER_PHONE;
        }
        updateAudioDeviceState();
    }

    /**
     * Construction.
     */


@@ 587,7 598,15 @@ public class AppRTCAudioManager {
    }

    public enum SpeakerPhonePreference {
        AUTO, EARPIECE, SPEAKER
        AUTO, EARPIECE, SPEAKER;

        public static SpeakerPhonePreference of(final Set<Media> media) {
            if (media.contains(Media.VIDEO)) {
                return SPEAKER;
            } else {
                return EARPIECE;
            }
        }
    }

    /**

A src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java => src/main/java/eu/siacs/conversations/xmpp/jingle/ContentAddition.java +88 -0
@@ 0,0 1,88 @@
package eu.siacs.conversations.xmpp.jingle;

import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableSet;

import java.util.Set;

import eu.siacs.conversations.xmpp.jingle.stanzas.Content;

public final class ContentAddition {

    public final Direction direction;
    public final Set<Summary> summary;

    private ContentAddition(Direction direction, Set<Summary> summary) {
        this.direction = direction;
        this.summary = summary;
    }

    public Set<Media> media() {
        return ImmutableSet.copyOf(Collections2.transform(summary, s -> s.media));
    }

    public static ContentAddition of(final Direction direction, final RtpContentMap rtpContentMap) {
        return new ContentAddition(direction, summary(rtpContentMap));
    }

    public static Set<Summary> summary(final RtpContentMap rtpContentMap) {
        return ImmutableSet.copyOf(
                Collections2.transform(
                        rtpContentMap.contents.entrySet(),
                        e -> {
                            final RtpContentMap.DescriptionTransport dt = e.getValue();
                            return new Summary(e.getKey(), dt.description.getMedia(), dt.senders);
                        }));
    }

    @Override
    public String toString() {
        return MoreObjects.toStringHelper(this)
                .add("direction", direction)
                .add("summary", summary)
                .toString();
    }

    public enum Direction {
        OUTGOING,
        INCOMING
    }

    public static final class Summary {
        public final String name;
        public final Media media;
        public final Content.Senders senders;

        private Summary(final String name, final Media media, final Content.Senders senders) {
            this.name = name;
            this.media = media;
            this.senders = senders;
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;
            if (o == null || getClass() != o.getClass()) return false;
            Summary summary = (Summary) o;
            return Objects.equal(name, summary.name)
                    && media == summary.media
                    && senders == summary.senders;
        }

        @Override
        public int hashCode() {
            return Objects.hashCode(name, media, senders);
        }

        @Override
        public String toString() {
            return MoreObjects.toStringHelper(this)
                    .add("name", name)
                    .add("media", media)
                    .add("senders", senders)
                    .toString();
        }
    }
}

M src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java => src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +510 -26
@@ 5,16 5,16 @@ import android.util.Log;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;

import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
import com.google.common.base.Stopwatch;
import com.google.common.base.Strings;
import com.google.common.base.Throwables;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.FutureCallback;


@@ 39,6 39,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;

import eu.siacs.conversations.BuildConfig;
import eu.siacs.conversations.Config;
import eu.siacs.conversations.crypto.axolotl.AxolotlService;
import eu.siacs.conversations.crypto.axolotl.CryptoFailedException;


@@ 53,6 54,7 @@ import eu.siacs.conversations.utils.IP;
import eu.siacs.conversations.xml.Element;
import eu.siacs.conversations.xml.Namespace;
import eu.siacs.conversations.xmpp.Jid;
import eu.siacs.conversations.xmpp.jingle.stanzas.Content;
import eu.siacs.conversations.xmpp.jingle.stanzas.Group;
import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo;
import eu.siacs.conversations.xmpp.jingle.stanzas.JinglePacket;


@@ 163,6 165,8 @@ public class JingleRtpConnection extends AbstractJingleConnection
    private Set<Media> proposedMedia;
    private RtpContentMap initiatorRtpContentMap;
    private RtpContentMap responderRtpContentMap;
    private RtpContentMap incomingContentAdd;
    private RtpContentMap outgoingContentAdd;
    private IceUdpTransportInfo.Setup peerDtlsSetup;
    private final Stopwatch sessionDuration = Stopwatch.createUnstarted();
    private final Queue<PeerConnection.PeerConnectionState> stateHistory = new LinkedList<>();


@@ 218,6 222,18 @@ public class JingleRtpConnection extends AbstractJingleConnection
            case SESSION_TERMINATE:
                receiveSessionTerminate(jinglePacket);
                break;
            case CONTENT_ADD:
                receiveContentAdd(jinglePacket);
                break;
            case CONTENT_ACCEPT:
                receiveContentAccept(jinglePacket);
                break;
            case CONTENT_REJECT:
                receiveContentReject(jinglePacket);
                break;
            case CONTENT_REMOVE:
                receiveContentRemove(jinglePacket);
                break;
            default:
                respondOk(jinglePacket);
                Log.d(


@@ 346,6 362,405 @@ public class JingleRtpConnection extends AbstractJingleConnection
        }
    }

    private void receiveContentAdd(final JinglePacket jinglePacket) {
        final RtpContentMap modification;
        try {
            modification = RtpContentMap.of(jinglePacket);
            modification.requireContentDescriptions();
        } catch (final RuntimeException e) {
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
                    Throwables.getRootCause(e));
            respondOk(jinglePacket);
            webRTCWrapper.close();
            sendSessionTerminate(Reason.of(e), e.getMessage());
            return;
        }
        if (isInState(State.SESSION_ACCEPTED)) {
            receiveContentAdd(jinglePacket, modification);
        } else {
            terminateWithOutOfOrder(jinglePacket);
        }
    }

    private void receiveContentAdd(
            final JinglePacket jinglePacket, final RtpContentMap modification) {
        final RtpContentMap remote = getRemoteContentMap();
        if (!Collections.disjoint(modification.getNames(), remote.getNames())) {
            respondOk(jinglePacket);
            this.webRTCWrapper.close();
            sendSessionTerminate(
                    Reason.FAILED_APPLICATION,
                    String.format(
                            "contents with names %s already exists",
                            Joiner.on(", ").join(modification.getNames())));
            return;
        }
        final ContentAddition contentAddition =
                ContentAddition.of(ContentAddition.Direction.INCOMING, modification);

        final RtpContentMap outgoing = this.outgoingContentAdd;
        final Set<ContentAddition.Summary> outgoingContentAddSummary =
                outgoing == null ? Collections.emptySet() : ContentAddition.summary(outgoing);

        if (outgoingContentAddSummary.equals(contentAddition.summary)) {
            if (isInitiator()) {
                Log.d(
                        Config.LOGTAG,
                        id.getAccount().getJid().asBareJid()
                                + ": respond with tie break to matching content-add offer");
                respondWithTieBreak(jinglePacket);
            } else {
                Log.d(
                        Config.LOGTAG,
                        id.getAccount().getJid().asBareJid()
                                + ": automatically accept matching content-add offer");
                acceptContentAdd(contentAddition.summary, modification);
            }
            return;
        }

        // once we can display multiple video tracks we can be more loose with this condition
        // theoretically it should also be fine to automatically accept audio only contents
        if (Media.audioOnly(remote.getMedia()) && Media.videoOnly(contentAddition.media())) {
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid() + ": received " + contentAddition);
            this.incomingContentAdd = modification;
            respondOk(jinglePacket);
            updateEndUserState();
        } else {
            respondOk(jinglePacket);
            // TODO do we want to add a reason?
            rejectContentAdd(modification);
        }
    }

    private void receiveContentAccept(final JinglePacket jinglePacket) {
        final RtpContentMap receivedContentAccept;
        try {
            receivedContentAccept = RtpContentMap.of(jinglePacket);
            receivedContentAccept.requireContentDescriptions();
        } catch (final RuntimeException e) {
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
                    Throwables.getRootCause(e));
            respondOk(jinglePacket);
            webRTCWrapper.close();
            sendSessionTerminate(Reason.of(e), e.getMessage());
            return;
        }

        final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
        if (outgoingContentAdd == null) {
            Log.d(Config.LOGTAG, "received content-accept when we had no outgoing content add");
            terminateWithOutOfOrder(jinglePacket);
            return;
        }
        final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
        if (ourSummary.equals(ContentAddition.summary(receivedContentAccept))) {
            this.outgoingContentAdd = null;
            respondOk(jinglePacket);
            receiveContentAccept(receivedContentAccept);
        } else {
            Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add");
            terminateWithOutOfOrder(jinglePacket);
        }
    }

    private void receiveContentAccept(final RtpContentMap receivedContentAccept) {
        final IceUdpTransportInfo.Setup peerDtlsSetup = getPeerDtlsSetup();
        final RtpContentMap modifiedContentMap =
                getRemoteContentMap().addContent(receivedContentAccept, peerDtlsSetup);

        setRemoteContentMap(modifiedContentMap);

        final SessionDescription answer = SessionDescription.of(modifiedContentMap, !isInitiator());

        final org.webrtc.SessionDescription sdp =
                new org.webrtc.SessionDescription(
                        org.webrtc.SessionDescription.Type.ANSWER, answer.toString());

        try {
            this.webRTCWrapper.setRemoteDescription(sdp).get();
        } catch (final Exception e) {
            final Throwable cause = Throwables.getRootCause(e);
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid()
                            + ": unable to set remote description after receiving content-accept",
                    cause);
            webRTCWrapper.close();
            sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
            return;
        }
        updateEndUserState();
        Log.d(
                Config.LOGTAG,
                id.getAccount().getJid().asBareJid()
                        + ": remote has accepted content-add "
                        + ContentAddition.summary(receivedContentAccept));
    }

    private void receiveContentReject(final JinglePacket jinglePacket) {
        final RtpContentMap receivedContentReject;
        try {
            receivedContentReject = RtpContentMap.of(jinglePacket);
        } catch (final RuntimeException e) {
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
                    Throwables.getRootCause(e));
            respondOk(jinglePacket);
            this.webRTCWrapper.close();
            sendSessionTerminate(Reason.of(e), e.getMessage());
            return;
        }

        final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
        if (outgoingContentAdd == null) {
            Log.d(Config.LOGTAG, "received content-reject when we had no outgoing content add");
            terminateWithOutOfOrder(jinglePacket);
            return;
        }
        final Set<ContentAddition.Summary> ourSummary = ContentAddition.summary(outgoingContentAdd);
        if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) {
            this.outgoingContentAdd = null;
            respondOk(jinglePacket);
            Log.d(Config.LOGTAG,jinglePacket.toString());
            receiveContentReject(ourSummary);
        } else {
            Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add");
            terminateWithOutOfOrder(jinglePacket);
        }
    }

    private void receiveContentReject(final Set<ContentAddition.Summary> summary) {
        try {
            this.webRTCWrapper.removeTrack(Media.VIDEO);
            final RtpContentMap localContentMap = customRollback();
            modifyLocalContentMap(localContentMap);
        } catch (final Exception e) {
            final Throwable cause = Throwables.getRootCause(e);
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid()
                            + ": unable to rollback local description after receiving content-reject",
                    cause);
            webRTCWrapper.close();
            sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
            return;
        }
        Log.d(
                Config.LOGTAG,
                id.getAccount().getJid().asBareJid()
                        + ": remote has rejected our content-add "
                        + summary);
    }

    private void receiveContentRemove(final JinglePacket jinglePacket) {
        final RtpContentMap receivedContentRemove;
        try {
            receivedContentRemove = RtpContentMap.of(jinglePacket);
            receivedContentRemove.requireContentDescriptions();
        } catch (final RuntimeException e) {
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid() + ": improperly formatted contents",
                    Throwables.getRootCause(e));
            respondOk(jinglePacket);
            this.webRTCWrapper.close();
            sendSessionTerminate(Reason.of(e), e.getMessage());
            return;
        }
        respondOk(jinglePacket);
        receiveContentRemove(receivedContentRemove);
    }

    private void receiveContentRemove(final RtpContentMap receivedContentRemove) {
        final RtpContentMap incomingContentAdd = this.incomingContentAdd;
        final Set<ContentAddition.Summary> contentAddSummary =
                incomingContentAdd == null
                        ? Collections.emptySet()
                        : ContentAddition.summary(incomingContentAdd);
        final Set<ContentAddition.Summary> removeSummary =
                ContentAddition.summary(receivedContentRemove);
        if (contentAddSummary.equals(removeSummary)) {
            this.incomingContentAdd = null;
            updateEndUserState();
        } else {
            webRTCWrapper.close();
            sendSessionTerminate(
                    Reason.FAILED_APPLICATION,
                    String.format(
                            "%s only supports %s as a means to retract a not yet accepted %s",
                            BuildConfig.APP_NAME,
                            JinglePacket.Action.CONTENT_REMOVE,
                            JinglePacket.Action.CONTENT_ACCEPT));
        }
    }

    public synchronized void retractContentAdd() {
        final RtpContentMap outgoingContentAdd = this.outgoingContentAdd;
        if (outgoingContentAdd == null) {
            throw new IllegalStateException("Not outgoing content add");
        }
        try {
            webRTCWrapper.removeTrack(Media.VIDEO);
            final RtpContentMap localContentMap = customRollback();
            modifyLocalContentMap(localContentMap);
        } catch (final Exception e) {
            final Throwable cause = Throwables.getRootCause(e);
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid()
                            + ": unable to rollback local description after trying to retract content-add",
                    cause);
            webRTCWrapper.close();
            sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage());
            return;
        }
        this.outgoingContentAdd = null;
        final JinglePacket retract =
                outgoingContentAdd
                        .toStub()
                        .toJinglePacket(JinglePacket.Action.CONTENT_REMOVE, id.sessionId);
        this.send(retract);
        Log.d(
                Config.LOGTAG,
                id.getAccount().getJid()
                        + ": retract content-add "
                        + ContentAddition.summary(outgoingContentAdd));
    }

    private RtpContentMap customRollback() throws ExecutionException, InterruptedException {
        final SessionDescription sdp = setLocalSessionDescription();
        final RtpContentMap localRtpContentMap = RtpContentMap.of(sdp, isInitiator());
        final SessionDescription answer = generateFakeResponse(localRtpContentMap);
        this.webRTCWrapper
                .setRemoteDescription(
                        new org.webrtc.SessionDescription(
                                org.webrtc.SessionDescription.Type.ANSWER, answer.toString()))
                .get();
        return localRtpContentMap;
    }

    private SessionDescription generateFakeResponse(final RtpContentMap localContentMap) {
        final RtpContentMap currentRemote = getRemoteContentMap();
        final RtpContentMap.Diff diff = currentRemote.diff(localContentMap);
        if (diff.isEmpty()) {
            throw new IllegalStateException(
                    "Unexpected rollback condition. No difference between local and remote");
        }
        final RtpContentMap patch = localContentMap.toContentModification(diff.added);
        if (ImmutableSet.of(Content.Senders.NONE).equals(patch.getSenders())) {
            final RtpContentMap nextRemote =
                    currentRemote.addContent(
                            patch.modifiedSenders(Content.Senders.NONE), getPeerDtlsSetup());
            return SessionDescription.of(nextRemote, !isInitiator());
        }
        throw new IllegalStateException(
                "Unexpected rollback condition. Senders were not uniformly none");
    }

    public synchronized void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition) {
        final RtpContentMap incomingContentAdd = this.incomingContentAdd;
        if (incomingContentAdd == null) {
            throw new IllegalStateException("No incoming content add");
        }

        if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) {
            this.incomingContentAdd = null;
            acceptContentAdd(contentAddition, incomingContentAdd);
        } else {
            throw new IllegalStateException("Accepted content add does not match pending content-add");
        }
    }

    private void acceptContentAdd(@NonNull final Set<ContentAddition.Summary> contentAddition, final RtpContentMap incomingContentAdd) {
        final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup();
        final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup);
        this.setRemoteContentMap(modifiedContentMap);

        final SessionDescription offer;
        try {
            offer = SessionDescription.of(modifiedContentMap, !isInitiator());
        } catch (final IllegalArgumentException | NullPointerException e) {
            Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e);
            webRTCWrapper.close();
            sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
            return;
        }
        this.incomingContentAdd = null;
        acceptContentAdd(contentAddition, offer);
    }

    private void acceptContentAdd(
            final Set<ContentAddition.Summary> contentAddition, final SessionDescription offer) {
        final org.webrtc.SessionDescription sdp =
                new org.webrtc.SessionDescription(
                        org.webrtc.SessionDescription.Type.OFFER, offer.toString());
        try {
            this.webRTCWrapper.setRemoteDescription(sdp).get();

            // TODO add tracks for 'media' where contentAddition.senders matches

            // TODO if senders.sending(isInitiator())

            this.webRTCWrapper.addTrack(Media.VIDEO);

            // TODO add additional transceivers for recv only cases

            final SessionDescription answer = setLocalSessionDescription();
            final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());

            final RtpContentMap contentAcceptMap =
                    rtpContentMap.toContentModification(
                            Collections2.transform(contentAddition, ca -> ca.name));
            Log.d(
                    Config.LOGTAG,
                    id.getAccount().getJid().asBareJid()
                            + ": sending content-accept "
                            + ContentAddition.summary(contentAcceptMap));
            modifyLocalContentMap(rtpContentMap);
            sendContentAccept(contentAcceptMap);
        } catch (final Exception e) {
            Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
            webRTCWrapper.close();
            sendSessionTerminate(Reason.FAILED_APPLICATION);
        }
    }

    private void sendContentAccept(final RtpContentMap contentAcceptMap) {
        final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId);
        send(jinglePacket);
    }

    public synchronized void rejectContentAdd() {
        final RtpContentMap incomingContentAdd = this.incomingContentAdd;
        if (incomingContentAdd == null) {
            throw new IllegalStateException("No incoming content add");
        }
        this.incomingContentAdd = null;
        updateEndUserState();
        rejectContentAdd(incomingContentAdd);
    }

    private void rejectContentAdd(final RtpContentMap contentMap) {
        final JinglePacket jinglePacket =
                contentMap
                        .toStub()
                        .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId);
        Log.d(
                Config.LOGTAG,
                id.getAccount().getJid().asBareJid()
                        + ": rejecting content "
                        + ContentAddition.summary(contentMap));
        send(jinglePacket);
    }

    private boolean checkForIceRestart(
            final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) {
        final RtpContentMap existing = getRemoteContentMap();


@@ 1534,6 1949,10 @@ public class JingleRtpConnection extends AbstractJingleConnection
                    return RtpEndUserState.CONNECTING;
                }
            case SESSION_ACCEPTED:
                final ContentAddition ca = getPendingContentAddition();
                if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) {
                    return RtpEndUserState.INCOMING_CONTENT_ADD;
                }
                return getPeerConnectionStateAsEndUserState();
            case REJECTED:
            case REJECTED_RACED:


@@ 1591,6 2010,18 @@ public class JingleRtpConnection extends AbstractJingleConnection
        }
    }

    public ContentAddition getPendingContentAddition() {
        final RtpContentMap in = this.incomingContentAdd;
        final RtpContentMap out = this.outgoingContentAdd;
        if (out != null) {
            return ContentAddition.of(ContentAddition.Direction.OUTGOING, out);
        } else if (in != null) {
            return ContentAddition.of(ContentAddition.Direction.INCOMING, in);
        } else {
            return null;
        }
    }

    public Set<Media> getMedia() {
        final State current = getState();
        if (current == State.NULL) {


@@ 1604,14 2035,16 @@ public class JingleRtpConnection extends AbstractJingleConnection
            return Preconditions.checkNotNull(
                    this.proposedMedia, "RTP connection has not been initialized properly");
        }
        final RtpContentMap localContentMap = getLocalContentMap();
        final RtpContentMap initiatorContentMap = initiatorRtpContentMap;
        if (initiatorContentMap != null) {
        if (localContentMap != null) {
            return localContentMap.getMedia();
        } else if (initiatorContentMap != null) {
            return initiatorContentMap.getMedia();
        } else if (isTerminated()) {
            return Collections.emptySet(); // we might fail before we ever got a chance to set media
            return Collections.emptySet(); //we might fail before we ever got a chance to set media
        } else {
            return Preconditions.checkNotNull(
                    this.proposedMedia, "RTP connection has not been initialized properly");
            return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly");
        }
    }



@@ 1625,6 2058,16 @@ public class JingleRtpConnection extends AbstractJingleConnection
        return status != null && status.isVerified();
    }

    public boolean addMedia(final Media media) {
        final Set<Media> currentMedia = getMedia();
        if (currentMedia.contains(media)) {
            throw new IllegalStateException(String.format("%s has already been proposed", media));
        }
        // TODO add state protection - can only add while ACCEPTED or so
        Log.d(Config.LOGTAG,"adding media: "+media);
        return webRTCWrapper.addTrack(media);
    }

    public synchronized void acceptCall() {
        switch (this.state) {
            case PROPOSED:


@@ 1743,17 2186,9 @@ public class JingleRtpConnection extends AbstractJingleConnection
        finish();
    }

    private void setupWebRTC(
            final Set<Media> media, final List<PeerConnection.IceServer> iceServers)
            throws WebRTCWrapper.InitializationException {
    private void setupWebRTC(final Set<Media> media, final List<PeerConnection.IceServer> iceServers) throws WebRTCWrapper.InitializationException {
        this.jingleConnectionManager.ensureConnectionIsRegistered(this);
        final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference;
        if (media.contains(Media.VIDEO)) {
            speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.SPEAKER;
        } else {
            speakerPhonePreference = AppRTCAudioManager.SpeakerPhonePreference.EARPIECE;
        }
        this.webRTCWrapper.setup(this.xmppConnectionService, speakerPhonePreference);
        this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media));
        this.webRTCWrapper.initializePeerConnection(media, iceServers);
    }



@@ 1905,21 2340,23 @@ public class JingleRtpConnection extends AbstractJingleConnection
                webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection);
                return;
            } else {
                webRTCWrapper.restartIce();
                this.restartIce();
            }
        }
        updateEndUserState();
    }

    private void restartIce() {
        this.stateHistory.clear();
        this.webRTCWrapper.restartIce();
    }

    @Override
    public void onRenegotiationNeeded() {
        this.webRTCWrapper.execute(this::renegotiate);
    }

    private void renegotiate() {
        //TODO needs to be called only for ice restarts; maybe in the call to restartICe()
        this.stateHistory.clear();
        this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false);
        final SessionDescription sessionDescription;
        try {
            sessionDescription = setLocalSessionDescription();


@@ 1945,19 2382,26 @@ public class JingleRtpConnection extends AbstractJingleConnection

        if (diff.hasModifications() && iceRestart) {
            webRTCWrapper.close();
            sendSessionTerminate(Reason.FAILED_APPLICATION, "WebRTC unexpectedly tried to modify content and transport at once");
            sendSessionTerminate(
                    Reason.FAILED_APPLICATION,
                    "WebRTC unexpectedly tried to modify content and transport at once");
            return;
        }

        if (iceRestart) {
            initiateIceRestart(rtpContentMap);
            return;
        } else if (diff.isEmpty()) {
            Log.d(
                    Config.LOGTAG,
                    "renegotiation. nothing to do. SignalingState="
                            + this.webRTCWrapper.getSignalingState());
        }

        if (diff.added.size() > 0) {
            sendContentAdd(rtpContentMap);
            modifyLocalContentMap(rtpContentMap);
            sendContentAdd(rtpContentMap, diff.added);
        }

    }

    private void initiateIceRestart(final RtpContentMap rtpContentMap) {


@@ 1977,8 2421,7 @@ public class JingleRtpConnection extends AbstractJingleConnection
                        return;
                    }
                    if (response.getType() == IqPacket.TYPE.ERROR) {
                        final Element error = response.findChild("error");
                        if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) {
                        if (isTieBreak(response)) {
                            Log.d(Config.LOGTAG, "received tie-break as result of ice restart");
                            return;
                        }


@@ 1990,8 2433,40 @@ public class JingleRtpConnection extends AbstractJingleConnection
                });
    }

    private void sendContentAdd(final RtpContentMap rtpContentMap) {
    private boolean isTieBreak(final IqPacket response) {
        final Element error = response.findChild("error");
        return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS);
    }

    private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection<String> added) {
        final RtpContentMap contentAdd = rtpContentMap.toContentModification(added);
        this.outgoingContentAdd = contentAdd;
        final JinglePacket jinglePacket =
                contentAdd.toJinglePacket(JinglePacket.Action.CONTENT_ADD, id.sessionId);
        jinglePacket.setTo(id.with);
        xmppConnectionService.sendIqPacket(
                id.account,
                jinglePacket,
                (connection, response) -> {
                    if (response.getType() == IqPacket.TYPE.RESULT) {
                        Log.d(
                                Config.LOGTAG,
                                id.getAccount().getJid().asBareJid()
                                        + ": received ACK to our content-add");
                        return;
                    }
                    if (response.getType() == IqPacket.TYPE.ERROR) {
                        if (isTieBreak(response)) {
                            this.outgoingContentAdd = null;
                            Log.d(Config.LOGTAG, "received tie-break as result of our content-add");
                            return;
                        }
                        handleIqErrorResponse(response);
                    }
                    if (response.getType() == IqPacket.TYPE.TIMEOUT) {
                        handleIqTimeoutResponse(response);
                    }
                });
    }

    private void setLocalContentMap(final RtpContentMap rtpContentMap) {


@@ 2010,6 2485,15 @@ public class JingleRtpConnection extends AbstractJingleConnection
        }
    }

    // this method is to be used for content map modifications that modify media
    private void modifyLocalContentMap(final RtpContentMap rtpContentMap) {
        final RtpContentMap activeContents = rtpContentMap.activeContents();
        setLocalContentMap(activeContents);
        this.webRTCWrapper.switchSpeakerPhonePreference(
                AppRTCAudioManager.SpeakerPhonePreference.of(activeContents.getMedia()));
        updateEndUserState();
    }

    private SessionDescription setLocalSessionDescription()
            throws ExecutionException, InterruptedException {
        final org.webrtc.SessionDescription sessionDescription =

M src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java => src/main/java/eu/siacs/conversations/xmpp/jingle/Media.java +15 -0
@@ 1,11 1,18 @@
package eu.siacs.conversations.xmpp.jingle;

import com.google.common.collect.ImmutableSet;

import java.util.Locale;
import java.util.Set;

import javax.annotation.Nonnull;

public enum Media {

    VIDEO, AUDIO, UNKNOWN;

    @Override
    @Nonnull
    public String toString() {
        return super.toString().toLowerCase(Locale.ROOT);
    }


@@ 17,4 24,12 @@ public enum Media {
            return UNKNOWN;
        }
    }

    public static boolean audioOnly(Set<Media> media) {
        return ImmutableSet.of(AUDIO).equals(media);
    }

    public static boolean videoOnly(Set<Media> media) {
        return ImmutableSet.of(VIDEO).equals(media);
    }
}

M src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java => src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java +55 -14
@@ 14,6 14,7 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;

import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;


@@ 92,6 93,10 @@ public class RtpContentMap {
                        }));
    }

    public Set<Content.Senders> getSenders() {
        return ImmutableSet.copyOf(Collections2.transform(contents.values(),dt -> dt.senders));
    }

    public List<String> getNames() {
        return ImmutableList.copyOf(contents.keySet());
    }


@@ 281,6 286,14 @@ public class RtpContentMap {
        return new RtpContentMap(this.group, contentMapBuilder.build());
    }

    public RtpContentMap modifiedSenders(final Content.Senders senders) {
        return new RtpContentMap(
                this.group,
                Maps.transformValues(
                        contents,
                        dt -> new DescriptionTransport(senders, dt.description, dt.transport)));
    }

    public RtpContentMap toContentModification(final Collection<String> modifications) {
        return new RtpContentMap(
                this.group,


@@ 291,6 304,22 @@ public class RtpContentMap {
                                        dt.senders, dt.description, IceUdpTransportInfo.STUB)));
    }

    public RtpContentMap toStub() {
        return new RtpContentMap(
                null,
                Maps.transformValues(
                        this.contents,
                        dt ->
                                new DescriptionTransport(
                                        dt.senders,
                                        RtpDescription.stub(dt.description.getMedia()),
                                        IceUdpTransportInfo.STUB)));
    }

    public RtpContentMap activeContents() {
        return new RtpContentMap(group, Maps.filterValues(this.contents, dt -> dt.senders != Content.Senders.NONE));
    }

    public Diff diff(final RtpContentMap rtpContentMap) {
        final Set<String> existingContentIds = this.contents.keySet();
        final Set<String> newContentIds = rtpContentMap.contents.keySet();


@@ 307,24 336,32 @@ public class RtpContentMap {
        }
    }

    public RtpContentMap addContent(final RtpContentMap modification) {
    public RtpContentMap addContent(
            final RtpContentMap modification, final IceUdpTransportInfo.Setup setup) {
        final IceUdpTransportInfo.Credentials credentials = getDistinctCredentials();
        final DTLS dtls = getDistinctDtls();
        final IceUdpTransportInfo iceUdpTransportInfo =
                IceUdpTransportInfo.of(credentials, dtls.setup, dtls.hash, dtls.fingerprint);
        final Map<String, DescriptionTransport> combined =
                new ImmutableMap.Builder<String, DescriptionTransport>()
                IceUdpTransportInfo.of(credentials, setup, dtls.hash, dtls.fingerprint);
        final Map<String, DescriptionTransport> combined = merge(contents, modification.contents);
                /*new ImmutableMap.Builder<String, DescriptionTransport>()
                        .putAll(contents)
                        .putAll(
                                Maps.transformValues(
                                        modification.contents,
                                        dt ->
                                                new DescriptionTransport(
                                                        dt.senders,
                                                        dt.description,
                                                        iceUdpTransportInfo)))
                        .build();
        return new RtpContentMap(modification.group, combined);
                        .putAll(modification.contents)
                        .build();*/
        final Map<String, DescriptionTransport> combinedFixedTransport =
                Maps.transformValues(
                        combined,
                        dt ->
                                new DescriptionTransport(
                                        dt.senders, dt.description, iceUdpTransportInfo));
        return new RtpContentMap(modification.group, combinedFixedTransport);
    }

    private static Map<String, DescriptionTransport> merge(
            final Map<String, DescriptionTransport> a, final Map<String, DescriptionTransport> b) {
        final Map<String, DescriptionTransport> combined = new HashMap<>();
        combined.putAll(a);
        combined.putAll(b);
        return ImmutableMap.copyOf(combined);
    }

    public static class DescriptionTransport {


@@ 410,6 447,10 @@ public class RtpContentMap {
            return !this.added.isEmpty() || !this.removed.isEmpty();
        }

        public boolean isEmpty() {
            return this.added.isEmpty() && this.removed.isEmpty();
        }

        @Override
        @Nonnull
        public String toString() {

M src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java => src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java +1 -0
@@ 5,6 5,7 @@ public enum RtpEndUserState {
    CONNECTING, //session-initiate or session-accepted but no webrtc peer connection yet
    CONNECTED, //session-accepted and webrtc peer connection is connected
    RECONNECTING, //session-accepted and webrtc peer connection was connected once but is currently disconnected or failed
    INCOMING_CONTENT_ADD, //session-accepted with a pending, incoming content-add
    FINDING_DEVICE, //'propose' has been sent out; no 184 ack yet
    RINGING, //'propose' has been sent out and it has been 184 acked
    ACCEPTING_CALL, //'proceed' message has been sent; but no session-initiate has been received

M src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java => src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java +1 -1
@@ 298,7 298,7 @@ public class SessionDescription {
            mediaAttributes.put("mid", name);

            mediaAttributes.put(descriptionTransport.senders.asMediaAttribute(isInitiatorContentMap), "");
            if (description.hasChild("rtcp-mux", Namespace.JINGLE_APPS_RTP)) {
            if (description.hasChild("rtcp-mux", Namespace.JINGLE_APPS_RTP) || group != null) {
                mediaAttributes.put("rtcp-mux", "");
            }


M src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java => src/main/java/eu/siacs/conversations/xmpp/jingle/ToneManager.java +27 -4
@@ 5,6 5,7 @@ import android.media.AudioManager;
import android.media.ToneGenerator;
import android.util.Log;

import java.util.Arrays;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;


@@ 19,6 20,7 @@ class ToneManager {
    private final Context context;

    private ToneState state = null;
    private RtpEndUserState endUserState = null;
    private ScheduledFuture<?> currentTone;
    private ScheduledFuture<?> currentResetFuture;
    private boolean appRtcAudioManagerHasControl = false;


@@ 51,7 53,11 @@ class ToneManager {
                return ToneState.ENDING_CALL;
            }
        }
        if (state == RtpEndUserState.CONNECTED || state == RtpEndUserState.RECONNECTING) {
        if (Arrays.asList(
                        RtpEndUserState.CONNECTED,
                        RtpEndUserState.RECONNECTING,
                        RtpEndUserState.INCOMING_CONTENT_ADD)
                .contains(state)) {
            if (media.contains(Media.VIDEO)) {
                return ToneState.NULL;
            } else {


@@ 62,14 68,19 @@ class ToneManager {
    }

    void transition(final RtpEndUserState state, final Set<Media> media) {
        transition(of(true, state, media), media);
        transition(state, of(true, state, media), media);
    }

    void transition(final boolean isInitiator, final RtpEndUserState state, final Set<Media> media) {
        transition(of(isInitiator, state, media), media);
        transition(state, of(isInitiator, state, media), media);
    }

    private synchronized void transition(ToneState state, final Set<Media> media) {
    private synchronized void transition(final RtpEndUserState endUserState, final ToneState state, final Set<Media> media) {
        final RtpEndUserState normalizeEndUserState = normalize(endUserState);
        if (this.endUserState == normalizeEndUserState) {
            return;
        }
        this.endUserState = normalizeEndUserState;
        if (this.state == state) {
            return;
        }


@@ 105,6 116,18 @@ class ToneManager {
        this.state = state;
    }

    private static RtpEndUserState normalize(final RtpEndUserState endUserState) {
        if (Arrays.asList(
                        RtpEndUserState.CONNECTED,
                        RtpEndUserState.RECONNECTING,
                        RtpEndUserState.INCOMING_CONTENT_ADD)
                .contains(endUserState)) {
            return RtpEndUserState.CONNECTED;
        } else {
            return endUserState;
        }
    }

    void setAppRtcAudioManagerHasControl(final boolean appRtcAudioManagerHasControl) {
        this.appRtcAudioManagerHasControl = appRtcAudioManagerHasControl;
    }

M src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java => src/main/java/eu/siacs/conversations/xmpp/jingle/TrackWrapper.java +49 -4
@@ 1,15 1,26 @@
package eu.siacs.conversations.xmpp.jingle;

import android.util.Log;

import com.google.common.base.CaseFormat;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;

import org.webrtc.MediaStreamTrack;
import org.webrtc.PeerConnection;
import org.webrtc.RtpSender;
import org.webrtc.RtpTransceiver;

import java.util.UUID;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import eu.siacs.conversations.Config;

class TrackWrapper<T extends MediaStreamTrack> {
    private final T track;
    private final RtpSender rtpSender;
    public final T track;
    public final RtpSender rtpSender;

    private TrackWrapper(final T track, final RtpSender rtpSender) {
        Preconditions.checkNotNull(track);


@@ 25,7 36,41 @@ class TrackWrapper<T extends MediaStreamTrack> {
    }

    public static <T extends MediaStreamTrack> Optional<T> get(
            final TrackWrapper<T> trackWrapper) {
        return trackWrapper == null ? Optional.absent() : Optional.of(trackWrapper.track);
            @Nullable final PeerConnection peerConnection, final TrackWrapper<T> trackWrapper) {
        if (trackWrapper == null) {
            return Optional.absent();
        }
        final RtpTransceiver transceiver =
                peerConnection == null ? null : getTransceiver(peerConnection, trackWrapper);
        if (transceiver == null) {
            Log.w(Config.LOGTAG, "unable to detect transceiver for " + trackWrapper.rtpSender.id());
            return Optional.of(trackWrapper.track);
        }
        final RtpTransceiver.RtpTransceiverDirection direction = transceiver.getDirection();
        if (direction == RtpTransceiver.RtpTransceiverDirection.SEND_ONLY
                || direction == RtpTransceiver.RtpTransceiverDirection.SEND_RECV) {
            return Optional.of(trackWrapper.track);
        } else {
            Log.d(Config.LOGTAG, "withholding track because transceiver is " + direction);
            return Optional.absent();
        }
    }

    public static <T extends MediaStreamTrack> RtpTransceiver getTransceiver(
            @Nonnull final PeerConnection peerConnection, final TrackWrapper<T> trackWrapper) {
        final RtpSender rtpSender = trackWrapper.rtpSender;
        for (final RtpTransceiver transceiver : peerConnection.getTransceivers()) {
            if (transceiver.getSender().id().equals(rtpSender.id())) {
                return transceiver;
            }
        }
        return null;
    }

    public static String id(final Class<? extends MediaStreamTrack> clazz) {
        return String.format(
                "%s-%s",
                CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, clazz.getSimpleName()),
                UUID.randomUUID().toString());
    }
}

M src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java => src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +65 -12
@@ 225,7 225,7 @@ public class WebRTCWrapper {

    public void setup(
            final XmppConnectionService service,
            final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference)
            @Nonnull final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference)
            throws InitializationException {
        try {
            PeerConnectionFactory.initialize(


@@ 330,18 330,35 @@ public class WebRTCWrapper {
        throw new IllegalStateException(String.format("Could not add track for %s", media));
    }

    public synchronized void removeTrack(final Media media) {
        if (media == Media.VIDEO) {
            removeVideoTrack(requirePeerConnection());
        }
    }

    private boolean addAudioTrack(final PeerConnection peerConnection) {
        final AudioSource audioSource =
                requirePeerConnectionFactory().createAudioSource(new MediaConstraints());
        final AudioTrack audioTrack =
                requirePeerConnectionFactory().createAudioTrack("my-audio-track", audioSource);
                requirePeerConnectionFactory()
                        .createAudioTrack(TrackWrapper.id(AudioTrack.class), audioSource);
        this.localAudioTrack = TrackWrapper.addTrack(peerConnection, audioTrack);
        return true;
    }

    private boolean addVideoTrack(final PeerConnection peerConnection) {
        Preconditions.checkState(
                this.localVideoTrack == null, "A local video track already exists");
        final TrackWrapper<VideoTrack> existing = this.localVideoTrack;
        if (existing != null) {
            final RtpTransceiver transceiver =
                    TrackWrapper.getTransceiver(peerConnection, existing);
            if (transceiver == null) {
                Log.w(EXTENDED_LOGGING_TAG, "unable to restart video transceiver");
                return false;
            }
            transceiver.setDirection(RtpTransceiver.RtpTransceiverDirection.SEND_RECV);
            this.videoSourceWrapper.startCapture();
            return true;
        }
        final VideoSourceWrapper videoSourceWrapper;
        try {
            videoSourceWrapper = initializeVideoSourceWrapper();


@@ 351,11 368,34 @@ public class WebRTCWrapper {
        }
        final VideoTrack videoTrack =
                requirePeerConnectionFactory()
                        .createVideoTrack("my-video-track", videoSourceWrapper.getVideoSource());
                        .createVideoTrack(
                                TrackWrapper.id(VideoTrack.class),
                                videoSourceWrapper.getVideoSource());
        this.localVideoTrack = TrackWrapper.addTrack(peerConnection, videoTrack);
        return true;
    }

    private void removeVideoTrack(final PeerConnection peerConnection) {
        final TrackWrapper<VideoTrack> localVideoTrack = this.localVideoTrack;
        if (localVideoTrack != null) {

            final RtpTransceiver exactTransceiver =
                    TrackWrapper.getTransceiver(peerConnection, localVideoTrack);
            if (exactTransceiver == null) {
                throw new IllegalStateException();
            }
            exactTransceiver.setDirection(RtpTransceiver.RtpTransceiverDirection.INACTIVE);
        }
        final VideoSourceWrapper videoSourceWrapper = this.videoSourceWrapper;
        if (videoSourceWrapper != null) {
            try {
                videoSourceWrapper.stopCapture();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }

    private static PeerConnection.RTCConfiguration buildConfiguration(
            final List<PeerConnection.IceServer> iceServers) {
        final PeerConnection.RTCConfiguration rtcConfig =


@@ 375,7 415,12 @@ public class WebRTCWrapper {
    }

    void restartIce() {
        executorService.execute(() -> requirePeerConnection().restartIce());
        executorService.execute(() -> {
            final PeerConnection peerConnection = requirePeerConnection();
            setIsReadyToReceiveIceCandidates(false);
            peerConnection.restartIce();
            requirePeerConnection().restartIce();}
        );
    }

    public void setIsReadyToReceiveIceCandidates(final boolean ready) {


@@ 450,7 495,8 @@ public class WebRTCWrapper {
    }

    boolean isMicrophoneEnabled() {
        final Optional<AudioTrack> audioTrack = TrackWrapper.get(this.localAudioTrack);
        final Optional<AudioTrack> audioTrack =
                TrackWrapper.get(peerConnection, this.localAudioTrack);
        if (audioTrack.isPresent()) {
            try {
                return audioTrack.get().enabled();


@@ 465,7 511,8 @@ public class WebRTCWrapper {
    }

    boolean setMicrophoneEnabled(final boolean enabled) {
        final Optional<AudioTrack> audioTrack = TrackWrapper.get(this.localAudioTrack);
        final Optional<AudioTrack> audioTrack =
                TrackWrapper.get(peerConnection, this.localAudioTrack);
        if (audioTrack.isPresent()) {
            try {
                audioTrack.get().setEnabled(enabled);


@@ 481,7 528,8 @@ public class WebRTCWrapper {
    }

    boolean isVideoEnabled() {
        final Optional<VideoTrack> videoTrack = TrackWrapper.get(this.localVideoTrack);
        final Optional<VideoTrack> videoTrack =
                TrackWrapper.get(peerConnection, this.localVideoTrack);
        if (videoTrack.isPresent()) {
            return videoTrack.get().enabled();
        }


@@ 489,7 537,8 @@ public class WebRTCWrapper {
    }

    void setVideoEnabled(final boolean enabled) {
        final Optional<VideoTrack> videoTrack = TrackWrapper.get(this.localVideoTrack);
        final Optional<VideoTrack> videoTrack =
                TrackWrapper.get(peerConnection, this.localVideoTrack);
        if (videoTrack.isPresent()) {
            videoTrack.get().setEnabled(enabled);
            return;


@@ 528,7 577,7 @@ public class WebRTCWrapper {
                MoreExecutors.directExecutor());
    }

    private static void logDescription(final SessionDescription sessionDescription) {
    public static void logDescription(final SessionDescription sessionDescription) {
        for (final String line :
                sessionDescription.description.split(
                        eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) {


@@ 612,7 661,7 @@ public class WebRTCWrapper {
    }

    Optional<VideoTrack> getLocalVideoTrack() {
        return TrackWrapper.get(this.localVideoTrack);
        return TrackWrapper.get(peerConnection, this.localVideoTrack);
    }

    Optional<VideoTrack> getRemoteVideoTrack() {


@@ 635,6 684,10 @@ public class WebRTCWrapper {
        executorService.execute(command);
    }

    public void switchSpeakerPhonePreference(AppRTCAudioManager.SpeakerPhonePreference preference) {
        mainHandler.post(() -> appRTCAudioManager.switchSpeakerPhonePreference(preference));
    }

    public interface EventCallback {
        void onIceCandidate(IceCandidate iceCandidate);


M src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java => src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/RtpDescription.java +89 -47
@@ 22,7 22,6 @@ import eu.siacs.conversations.xmpp.jingle.SessionDescription;

public class RtpDescription extends GenericDescription {


    private RtpDescription(final String media) {
        super("description", Namespace.JINGLE_APPS_RTP);
        this.setAttribute("media", media);


@@ 32,6 31,10 @@ public class RtpDescription extends GenericDescription {
        super("description", Namespace.JINGLE_APPS_RTP);
    }

    public static RtpDescription stub(final Media media) {
        return new RtpDescription(media.toString());
    }

    public Media getMedia() {
        return Media.of(this.getAttribute("media"));
    }


@@ 57,7 60,8 @@ public class RtpDescription extends GenericDescription {
    public List<RtpHeaderExtension> getHeaderExtensions() {
        final ImmutableList.Builder<RtpHeaderExtension> builder = new ImmutableList.Builder<>();
        for (final Element child : getChildren()) {
            if ("rtp-hdrext".equals(child.getName()) && Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(child.getNamespace())) {
            if ("rtp-hdrext".equals(child.getName())
                    && Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(child.getNamespace())) {
                builder.add(RtpHeaderExtension.upgrade(child));
            }
        }


@@ 67,7 71,9 @@ public class RtpDescription extends GenericDescription {
    public List<Source> getSources() {
        final ImmutableList.Builder<Source> builder = new ImmutableList.Builder<>();
        for (final Element child : this.children) {
            if ("source".equals(child.getName()) && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(child.getNamespace())) {
            if ("source".equals(child.getName())
                    && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(
                            child.getNamespace())) {
                builder.add(Source.upgrade(child));
            }
        }


@@ 77,7 83,9 @@ public class RtpDescription extends GenericDescription {
    public List<SourceGroup> getSourceGroups() {
        final ImmutableList.Builder<SourceGroup> builder = new ImmutableList.Builder<>();
        for (final Element child : this.children) {
            if ("ssrc-group".equals(child.getName()) && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(child.getNamespace())) {
            if ("ssrc-group".equals(child.getName())
                    && Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(
                            child.getNamespace())) {
                builder.add(SourceGroup.upgrade(child));
            }
        }


@@ 85,8 93,12 @@ public class RtpDescription extends GenericDescription {
    }

    public static RtpDescription upgrade(final Element element) {
        Preconditions.checkArgument("description".equals(element.getName()), "Name of provided element is not description");
        Preconditions.checkArgument(Namespace.JINGLE_APPS_RTP.equals(element.getNamespace()), "Element does not match the jingle rtp namespace");
        Preconditions.checkArgument(
                "description".equals(element.getName()),
                "Name of provided element is not description");
        Preconditions.checkArgument(
                Namespace.JINGLE_APPS_RTP.equals(element.getNamespace()),
                "Element does not match the jingle rtp namespace");
        final RtpDescription description = new RtpDescription();
        description.setAttributes(element.getAttributes());
        description.setChildren(element.getChildren());


@@ 116,7 128,8 @@ public class RtpDescription extends GenericDescription {

        private static FeedbackNegotiation upgrade(final Element element) {
            Preconditions.checkArgument("rtcp-fb".equals(element.getName()));
            Preconditions.checkArgument(Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace()));
            Preconditions.checkArgument(
                    Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace()));
            final FeedbackNegotiation feedback = new FeedbackNegotiation();
            feedback.setAttributes(element.getAttributes());
            feedback.setChildren(element.getChildren());


@@ 126,13 139,13 @@ public class RtpDescription extends GenericDescription {
        public static List<FeedbackNegotiation> fromChildren(final List<Element> children) {
            ImmutableList.Builder<FeedbackNegotiation> builder = new ImmutableList.Builder<>();
            for (final Element child : children) {
                if ("rtcp-fb".equals(child.getName()) && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) {
                if ("rtcp-fb".equals(child.getName())
                        && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) {
                    builder.add(upgrade(child));
                }
            }
            return builder.build();
        }

    }

    public static class FeedbackNegotiationTrrInt extends Element {


@@ 142,7 155,6 @@ public class RtpDescription extends GenericDescription {
            this.setAttribute("value", value);
        }


        private FeedbackNegotiationTrrInt() {
            super("rtcp-fb-trr-int", Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION);
        }


@@ 150,12 162,12 @@ public class RtpDescription extends GenericDescription {
        public int getValue() {
            final String value = getAttribute("value");
            return Integer.parseInt(value);

        }

        private static FeedbackNegotiationTrrInt upgrade(final Element element) {
            Preconditions.checkArgument("rtcp-fb-trr-int".equals(element.getName()));
            Preconditions.checkArgument(Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace()));
            Preconditions.checkArgument(
                    Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(element.getNamespace()));
            final FeedbackNegotiationTrrInt trr = new FeedbackNegotiationTrrInt();
            trr.setAttributes(element.getAttributes());
            trr.setChildren(element.getChildren());


@@ 163,9 175,11 @@ public class RtpDescription extends GenericDescription {
        }

        public static List<FeedbackNegotiationTrrInt> fromChildren(final List<Element> children) {
            ImmutableList.Builder<FeedbackNegotiationTrrInt> builder = new ImmutableList.Builder<>();
            ImmutableList.Builder<FeedbackNegotiationTrrInt> builder =
                    new ImmutableList.Builder<>();
            for (final Element child : children) {
                if ("rtcp-fb-trr-int".equals(child.getName()) && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) {
                if ("rtcp-fb-trr-int".equals(child.getName())
                        && Namespace.JINGLE_RTP_FEEDBACK_NEGOTIATION.equals(child.getNamespace())) {
                    builder.add(upgrade(child));
                }
            }


@@ 173,9 187,8 @@ public class RtpDescription extends GenericDescription {
        }
    }


    //XEP-0294: Jingle RTP Header Extensions Negotiation
    //maps to `extmap:$id $uri`
    // XEP-0294: Jingle RTP Header Extensions Negotiation
    // maps to `extmap:$id $uri`
    public static class RtpHeaderExtension extends Element {

        private RtpHeaderExtension() {


@@ 198,7 211,8 @@ public class RtpDescription extends GenericDescription {

        public static RtpHeaderExtension upgrade(final Element element) {
            Preconditions.checkArgument("rtp-hdrext".equals(element.getName()));
            Preconditions.checkArgument(Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(element.getNamespace()));
            Preconditions.checkArgument(
                    Namespace.JINGLE_RTP_HEADER_EXTENSIONS.equals(element.getNamespace()));
            final RtpHeaderExtension extension = new RtpHeaderExtension();
            extension.setAttributes(element.getAttributes());
            extension.setChildren(element.getChildren());


@@ 217,7 231,7 @@ public class RtpDescription extends GenericDescription {
        }
    }

    //maps to `rtpmap:$id $name/$clockrate/$channels`
    // maps to `rtpmap:$id $name/$clockrate/$channels`
    public static class PayloadType extends Element {

        private PayloadType() {


@@ 238,8 252,14 @@ public class RtpDescription extends GenericDescription {
            final int channels = getChannels();
            final String name = getPayloadTypeName();
            Preconditions.checkArgument(name != null, "Payload-type name must not be empty");
            SessionDescription.checkNoWhitespace(name, "payload-type name must not contain whitespaces");
            return getId() + " " + name + "/" + getClockRate() + (channels == 1 ? "" : "/" + channels);
            SessionDescription.checkNoWhitespace(
                    name, "payload-type name must not contain whitespaces");
            return getId()
                    + " "
                    + name
                    + "/"
                    + getClockRate()
                    + (channels == 1 ? "" : "/" + channels);
        }

        public int getIntId() {


@@ 251,7 271,6 @@ public class RtpDescription extends GenericDescription {
            return this.getAttribute("id");
        }


        public String getPayloadTypeName() {
            return this.getAttribute("name");
        }


@@ 271,7 290,8 @@ public class RtpDescription extends GenericDescription {
        public int getChannels() {
            final String channels = this.getAttribute("channels");
            if (channels == null) {
                return 1; // The number of channels; if omitted, it MUST be assumed to contain one channel
                return 1; // The number of channels; if omitted, it MUST be assumed to contain one
                          // channel
            }
            try {
                return Integer.parseInt(channels);


@@ 299,7 319,9 @@ public class RtpDescription extends GenericDescription {
        }

        public static PayloadType of(final Element element) {
            Preconditions.checkArgument("payload-type".equals(element.getName()), "element name must be called payload-type");
            Preconditions.checkArgument(
                    "payload-type".equals(element.getName()),
                    "element name must be called payload-type");
            PayloadType payloadType = new PayloadType();
            payloadType.setAttributes(element.getAttributes());
            payloadType.setChildren(element.getChildren());


@@ 339,8 361,8 @@ public class RtpDescription extends GenericDescription {
        }
    }

    //map to `fmtp $id key=value;key=value
    //where id is the id of the parent payload-type
    // map to `fmtp $id key=value;key=value
    // where id is the id of the parent payload-type
    public static class Parameter extends Element {

        private Parameter() {


@@ 362,7 384,8 @@ public class RtpDescription extends GenericDescription {
        }

        public static Parameter of(final Element element) {
            Preconditions.checkArgument("parameter".equals(element.getName()), "element name must be called parameter");
            Preconditions.checkArgument(
                    "parameter".equals(element.getName()), "element name must be called parameter");
            Parameter parameter = new Parameter();
            parameter.setAttributes(element.getAttributes());
            parameter.setChildren(element.getChildren());


@@ 375,12 398,18 @@ public class RtpDescription extends GenericDescription {
            for (int i = 0; i < parameters.size(); ++i) {
                final Parameter p = parameters.get(i);
                final String name = p.getParameterName();
                Preconditions.checkArgument(name != null, String.format("parameter for %s must have a name", id));
                SessionDescription.checkNoWhitespace(name, String.format("parameter names for %s must not contain whitespaces", id));
                Preconditions.checkArgument(
                        name != null, String.format("parameter for %s must have a name", id));
                SessionDescription.checkNoWhitespace(
                        name,
                        String.format("parameter names for %s must not contain whitespaces", id));

                final String value = p.getParameterValue();
                Preconditions.checkArgument(value != null, String.format("parameter for %s must have a value", id));
                SessionDescription.checkNoWhitespace(value, String.format("parameter values for %s must not contain whitespaces", id));
                Preconditions.checkArgument(
                        value != null, String.format("parameter for %s must have a value", id));
                SessionDescription.checkNoWhitespace(
                        value,
                        String.format("parameter values for %s must not contain whitespaces", id));

                stringBuilder.append(name).append('=').append(value);
                if (i != parameters.size() - 1) {


@@ 393,8 422,11 @@ public class RtpDescription extends GenericDescription {
        public static String toSdpString(final String id, final Parameter parameter) {
            final String name = parameter.getParameterName();
            final String value = parameter.getParameterValue();
            Preconditions.checkArgument(value != null, String.format("parameter for %s must have a value", id));
            SessionDescription.checkNoWhitespace(value, String.format("parameter values for %s must not contain whitespaces", id));
            Preconditions.checkArgument(
                    value != null, String.format("parameter for %s must have a value", id));
            SessionDescription.checkNoWhitespace(
                    value,
                    String.format("parameter values for %s must not contain whitespaces", id));
            if (Strings.isNullOrEmpty(name)) {
                return String.format("%s %s", id, value);
            } else {


@@ 420,8 452,8 @@ public class RtpDescription extends GenericDescription {
        }
    }

    //XEP-0339: Source-Specific Media Attributes in Jingle
    //maps to `a=ssrc:<ssrc-id> <attribute>:<value>`
    // XEP-0339: Source-Specific Media Attributes in Jingle
    // maps to `a=ssrc:<ssrc-id> <attribute>:<value>`
    public static class Source extends Element {

        private Source() {


@@ 452,7 484,9 @@ public class RtpDescription extends GenericDescription {

        public static Source upgrade(final Element element) {
            Preconditions.checkArgument("source".equals(element.getName()));
            Preconditions.checkArgument(Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(element.getNamespace()));
            Preconditions.checkArgument(
                    Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(
                            element.getNamespace()));
            final Source source = new Source();
            source.setChildren(element.getChildren());
            source.setAttributes(element.getAttributes());


@@ 489,7 523,6 @@ public class RtpDescription extends GenericDescription {
                return parameter;
            }
        }

    }

    public static class SourceGroup extends Element {


@@ 525,7 558,9 @@ public class RtpDescription extends GenericDescription {

        public static SourceGroup upgrade(final Element element) {
            Preconditions.checkArgument("ssrc-group".equals(element.getName()));
            Preconditions.checkArgument(Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(element.getNamespace()));
            Preconditions.checkArgument(
                    Namespace.JINGLE_RTP_SOURCE_SPECIFIC_MEDIA_ATTRIBUTES.equals(
                            element.getNamespace()));
            final SourceGroup group = new SourceGroup();
            group.setChildren(element.getChildren());
            group.setAttributes(element.getAttributes());


@@ 533,15 568,18 @@ public class RtpDescription extends GenericDescription {
        }
    }

    public static RtpDescription of(final SessionDescription sessionDescription, final SessionDescription.Media media) {
    public static RtpDescription of(
            final SessionDescription sessionDescription, final SessionDescription.Media media) {
        final RtpDescription rtpDescription = new RtpDescription(media.media);
        final Map<String, List<Parameter>> parameterMap = new HashMap<>();
        final ArrayListMultimap<String, Element> feedbackNegotiationMap = ArrayListMultimap.create();
        final ArrayListMultimap<String, Source.Parameter> sourceParameterMap = ArrayListMultimap.create();
        final Set<String> attributes = Sets.newHashSet(Iterables.concat(
                sessionDescription.attributes.keySet(),
                media.attributes.keySet()
        ));
        final ArrayListMultimap<String, Element> feedbackNegotiationMap =
                ArrayListMultimap.create();
        final ArrayListMultimap<String, Source.Parameter> sourceParameterMap =
                ArrayListMultimap.create();
        final Set<String> attributes =
                Sets.newHashSet(
                        Iterables.concat(
                                sessionDescription.attributes.keySet(), media.attributes.keySet()));
        for (final String rtcpFb : media.attributes.get("rtcp-fb")) {
            final String[] parts = rtcpFb.split(" ");
            if (parts.length >= 2) {


@@ 550,7 588,10 @@ public class RtpDescription extends GenericDescription {
                final String subType = parts.length >= 3 ? parts[2] : null;
                if ("trr-int".equals(type)) {
                    if (subType != null) {
                        feedbackNegotiationMap.put(id, new FeedbackNegotiationTrrInt(SessionDescription.ignorantIntParser(subType)));
                        feedbackNegotiationMap.put(
                                id,
                                new FeedbackNegotiationTrrInt(
                                        SessionDescription.ignorantIntParser(subType)));
                    }
                } else {
                    feedbackNegotiationMap.put(id, new FeedbackNegotiation(type, subType));


@@ 602,7 643,8 @@ public class RtpDescription extends GenericDescription {
                rtpDescription.addChild(new SourceGroup(semantics, builder.build()));
            }
        }
        for (Map.Entry<String, Collection<Source.Parameter>> source : sourceParameterMap.asMap().entrySet()) {
        for (Map.Entry<String, Collection<Source.Parameter>> source :
                sourceParameterMap.asMap().entrySet()) {
            rtpDescription.addChild(new Source(source.getKey(), source.getValue()));
        }
        if (media.attributes.containsKey("rtcp-mux")) {