~singpolyma/cheogram-android

8dc854b8507bdce08db410d6802eb2a08239d14d — Daniel Gultsch 4 months ago 2d616dd
handle senders modification via content-modify

Dino uses this to enable/disable video when a video content is already present
M src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java => src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java +3 -1
@@ 250,7 250,9 @@ public class JingleConnectionManager extends AbstractConnectionManager {
        final Element error = response.addChild("error");
        error.setAttribute("type", conditionType);
        error.addChild(condition, "urn:ietf:params:xml:ns:xmpp-stanzas");
        error.addChild(jingleCondition, Namespace.JINGLE_ERRORS);
        if (jingleCondition != null) {
            error.addChild(jingleCondition, Namespace.JINGLE_ERRORS);
        }
        account.getXmppConnection().sendIqPacket(response, null);
    }


M src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java => src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +77 -9
@@ 15,8 15,11 @@ import com.google.common.base.Throwables;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
import com.google.common.util.concurrent.FutureCallback;


@@ 522,15 525,17 @@ public class JingleRtpConnection extends AbstractJingleConnection
    }

    private void receiveContentModify(final JinglePacket jinglePacket) {
        // TODO check session accepted
        final Map<String, Content.Senders> modification =
                Maps.transformEntries(
                        jinglePacket.getJingleContents(), (key, value) -> value.getSenders());
        respondOk(jinglePacket);
        final boolean isInitiator = isInitiator();
        final RtpContentMap currentOutgoing = this.outgoingContentAdd;
        final RtpContentMap remoteContentMap = this.getRemoteContentMap();
        final Set<String> currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet();
        Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")");
        if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) {
            final boolean isInitiator = isInitiator();
            respondOk(jinglePacket);
            final RtpContentMap modifiedContentMap;
            try {
                modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification);


@@ 541,18 546,72 @@ public class JingleRtpConnection extends AbstractJingleConnection
            }
            this.outgoingContentAdd = modifiedContentMap;
            Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add");
        } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) {
            respondOk(jinglePacket);
            final RtpContentMap modifiedRemoteContentMap;
            try {
                modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification);
            } catch (final IllegalArgumentException e) {
                webRTCWrapper.close();
                sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
                return;
            }
            final SessionDescription offer;
            try {
                offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator());
            } catch (final IllegalArgumentException | NullPointerException e) {
                Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e);
                webRTCWrapper.close();
                sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage());
                return;
            }
            Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification");
            this.autoAcceptContentModify(modifiedRemoteContentMap, offer);
        } else {
            Log.d(Config.LOGTAG,"received unsupported content modification "+modification);
            respondWithItemNotFound(jinglePacket);
        }
    }

    private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) {
        this.setRemoteContentMap(modifiedRemoteContentMap);
        final org.webrtc.SessionDescription sdp =
                new org.webrtc.SessionDescription(
                        org.webrtc.SessionDescription.Type.OFFER, offer.toString());
        try {
            this.webRTCWrapper.setRemoteDescription(sdp).get();
            // auto accept is only done when we already have tracks
            final SessionDescription answer = setLocalSessionDescription();
            final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator());
            modifyLocalContentMap(rtpContentMap);
            // we do not need to send an answer but do we have to resend the candidates currently in SDP?
            //resendCandidatesFromSdp(answer);
            webRTCWrapper.setIsReadyToReceiveIceCandidates(true);
        } catch (final Exception e) {
            Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e));
            webRTCWrapper.close();
            sendSessionTerminate(
                    Reason.FAILED_APPLICATION,
                    String.format(
                            "%s only supports %s as a means to modify a not yet accepted %s",
                            BuildConfig.APP_NAME,
                            JinglePacket.Action.CONTENT_MODIFY,
                            JinglePacket.Action.CONTENT_ADD));
            sendSessionTerminate(Reason.FAILED_APPLICATION);
        }
    }

    private void resendCandidatesFromSdp(final SessionDescription answer) {
        final ImmutableMultimap.Builder<String, IceUdpTransportInfo.Candidate> candidateBuilder = new ImmutableMultimap.Builder<>();
        for(final SessionDescription.Media media : answer.media) {
            final String mid = Iterables.getFirst(media.attributes.get("mid"), null);
            if (Strings.isNullOrEmpty(mid)) {
                continue;
            }
            for(final String sdpCandidate : media.attributes.get("candidate")) {
                final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null);
                if (candidate != null) {
                    candidateBuilder.put(mid,candidate);
                }
            }
        }
        final ImmutableMultimap<String, IceUdpTransportInfo.Candidate> candidates = candidateBuilder.build();
        sendTransportInfo(candidates);
    }

    private void receiveContentReject(final JinglePacket jinglePacket) {
        final RtpContentMap receivedContentReject;
        try {


@@ 1877,6 1936,11 @@ public class JingleRtpConnection extends AbstractJingleConnection
        send(jinglePacket);
    }

    private void sendTransportInfo(final Multimap<String, IceUdpTransportInfo.Candidate> candidates) {
        // TODO send all candidates in one transport-info
    }


    private void send(final JinglePacket jinglePacket) {
        jinglePacket.setTo(id.with);
        xmppConnectionService.sendIqPacket(id.account, jinglePacket, this::handleIqResponse);


@@ 1963,6 2027,10 @@ public class JingleRtpConnection extends AbstractJingleConnection
        respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait");
    }

    private void respondWithItemNotFound(final JinglePacket jinglePacket) {
        respondWithJingleError(jinglePacket, null, "item-not-found", "cancel");
    }

    void respondWithJingleError(
            final IqPacket original,
            String jingleCondition,

M src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java => src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +36 -31
@@ 200,41 200,46 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
        public static Candidate fromSdpAttribute(final String attribute, String currentUfrag) {
            final String[] pair = attribute.split(":", 2);
            if (pair.length == 2 && "candidate".equals(pair[0])) {
                final String[] segments = pair[1].split(" ");
                if (segments.length >= 6) {
                    final String id = UUID.randomUUID().toString();
                    final String foundation = segments[0];
                    final String component = segments[1];
                    final String transport = segments[2].toLowerCase(Locale.ROOT);
                    final String priority = segments[3];
                    final String connectionAddress = segments[4];
                    final String port = segments[5];
                    final HashMap<String, String> additional = new HashMap<>();
                    for (int i = 6; i < segments.length - 1; i = i + 2) {
                        additional.put(segments[i], segments[i + 1]);
                    }
                    final String ufrag = additional.get("ufrag");
                    if (ufrag != null && !ufrag.equals(currentUfrag)) {
                        return null;
                    }
                    final Candidate candidate = new Candidate();
                    candidate.setAttribute("component", component);
                    candidate.setAttribute("foundation", foundation);
                    candidate.setAttribute("generation", additional.get("generation"));
                    candidate.setAttribute("rel-addr", additional.get("raddr"));
                    candidate.setAttribute("rel-port", additional.get("rport"));
                    candidate.setAttribute("id", id);
                    candidate.setAttribute("ip", connectionAddress);
                    candidate.setAttribute("port", port);
                    candidate.setAttribute("priority", priority);
                    candidate.setAttribute("protocol", transport);
                    candidate.setAttribute("type", additional.get("typ"));
                    return candidate;
                }
                return fromSdpAttributeValue(pair[1], currentUfrag);
            }
            return null;
        }

        public static Candidate fromSdpAttributeValue(final String value, final String currentUfrag) {
            final String[] segments = value.split(" ");
            if (segments.length < 6) {
                return null;
            }
            final String id = UUID.randomUUID().toString();
            final String foundation = segments[0];
            final String component = segments[1];
            final String transport = segments[2].toLowerCase(Locale.ROOT);
            final String priority = segments[3];
            final String connectionAddress = segments[4];
            final String port = segments[5];
            final HashMap<String, String> additional = new HashMap<>();
            for (int i = 6; i < segments.length - 1; i = i + 2) {
                additional.put(segments[i], segments[i + 1]);
            }
            final String ufrag = additional.get("ufrag");
            if (currentUfrag != null && ufrag != null && !ufrag.equals(currentUfrag)) {
                return null;
            }
            final Candidate candidate = new Candidate();
            candidate.setAttribute("component", component);
            candidate.setAttribute("foundation", foundation);
            candidate.setAttribute("generation", additional.get("generation"));
            candidate.setAttribute("rel-addr", additional.get("raddr"));
            candidate.setAttribute("rel-port", additional.get("rport"));
            candidate.setAttribute("id", id);
            candidate.setAttribute("ip", connectionAddress);
            candidate.setAttribute("port", port);
            candidate.setAttribute("priority", priority);
            candidate.setAttribute("protocol", transport);
            candidate.setAttribute("type", additional.get("typ"));
            return candidate;
        }

        public int getComponent() {
            return getAttributeAsInt("component");
        }