From ad65f51f7af6ebb6478ff06688415f6e9623e5b5 Mon Sep 17 00:00:00 2001 From: Stephen Paul Weber Date: Mon, 7 Mar 2022 16:21:28 -0500 Subject: [PATCH] Revert stuff that needs newer libwebrtc --- build.gradle | 2 +- .../xmpp/jingle/JingleRtpConnection.java | 647 +++++------------- .../xmpp/jingle/RtpContentMap.java | 99 +-- .../xmpp/jingle/SessionDescription.java | 5 +- .../xmpp/jingle/WebRTCWrapper.java | 170 +++-- .../jingle/stanzas/IceUdpTransportInfo.java | 91 +-- 6 files changed, 259 insertions(+), 755 deletions(-) diff --git a/build.gradle b/build.gradle index c33b5646a..85324e3a8 100644 --- a/build.gradle +++ b/build.gradle @@ -103,7 +103,7 @@ dependencies { implementation 'io.michaelrocks:libphonenumber-android:8.12.36' implementation 'io.github.nishkarsh:android-permissions:2.1.6' implementation 'androidx.recyclerview:recyclerview:1.1.0' - implementation urlFile('https://cloudflare-ipfs.com/ipfs/QmeqMiLxHi8AAjXobxr3QTfa1bSSLyAu86YviAqQnjxCjM/libwebrtc.aar', 'libwebrtc.aar') + implementation 'org.webrtc:google-webrtc:1.0.32006' // INSERT } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index c93e323e5..1a262ffa4 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -1,5 +1,6 @@ package eu.siacs.conversations.xmpp.jingle; +import android.os.SystemClock; import android.util.Log; import androidx.annotation.NonNull; @@ -20,21 +21,18 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import org.webrtc.DtmfSender; import org.webrtc.EglBase; import org.webrtc.IceCandidate; import org.webrtc.PeerConnection; import org.webrtc.VideoTrack; +import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.Set; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -154,17 +152,16 @@ public class JingleRtpConnection extends AbstractJingleConnection } private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this); - private final Queue> - pendingIceCandidates = new LinkedList<>(); + private final ArrayDeque>> pendingIceCandidates = new ArrayDeque<>(); private final OmemoVerification omemoVerification = new OmemoVerification(); private final Message message; private State state = State.NULL; private Set proposedMedia; private RtpContentMap initiatorRtpContentMap; private RtpContentMap responderRtpContentMap; - private IceUdpTransportInfo.Setup peerDtlsSetup; private final Stopwatch sessionDuration = Stopwatch.createUnstarted(); - private final Queue stateHistory = new LinkedList<>(); + private long rtpConnectionStarted = 0; //time of 'connected' + private long rtpConnectionEnded = 0; private ScheduledFuture ringingTimeoutFuture; JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) { @@ -204,6 +201,7 @@ public class JingleRtpConnection extends AbstractJingleConnection @Override synchronized void deliverPacket(final JinglePacket jinglePacket) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection"); switch (jinglePacket.getAction()) { case SESSION_INITIATE: receiveSessionInitiate(jinglePacket); @@ -286,27 +284,26 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void receiveTransportInfo(final JinglePacket jinglePacket) { - // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to - // INITIALIZED only after transport-info has been received - if (isInState( - State.NULL, - State.PROCEED, - State.SESSION_INITIALIZED, - State.SESSION_INITIALIZED_PRE_APPROVED, - State.SESSION_ACCEPTED)) { + //Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to INITIALIZED only after transport-info has been received + if (isInState(State.NULL, State.PROCEED, State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) { + respondOk(jinglePacket); final RtpContentMap contentMap; try { contentMap = RtpContentMap.of(jinglePacket); - } catch (final IllegalArgumentException | NullPointerException e) { - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": improperly formatted contents; ignoring", - e); - respondOk(jinglePacket); + } catch (IllegalArgumentException | NullPointerException e) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e); return; } - receiveTransportInfo(jinglePacket, contentMap); + final Set> candidates = contentMap.contents.entrySet(); + if (this.state == State.SESSION_ACCEPTED) { + try { + processCandidates(candidates); + } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { + Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored"); + } + } else { + pendingIceCandidates.push(candidates); + } } else { if (isTerminated()) { respondOk(jinglePacket); @@ -325,186 +322,8 @@ public class JingleRtpConnection extends AbstractJingleConnection } } - private void receiveTransportInfo( - final JinglePacket jinglePacket, final RtpContentMap contentMap) { - final Set> candidates = - contentMap.contents.entrySet(); - if (this.state == State.SESSION_ACCEPTED) { - // zero candidates + modified credentials are an ICE restart offer - if (checkForIceRestart(jinglePacket, contentMap)) { - return; - } - respondOk(jinglePacket); - try { - processCandidates(candidates); - } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { - Log.w( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored"); - } - } else { - respondOk(jinglePacket); - pendingIceCandidates.addAll(candidates); - } - } - - private boolean checkForIceRestart( - final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) { - final RtpContentMap existing = getRemoteContentMap(); - final Set existingCredentials; - final IceUdpTransportInfo.Credentials newCredentials; - try { - existingCredentials = existing.getCredentials(); - newCredentials = rtpContentMap.getDistinctCredentials(); - } catch (final IllegalStateException e) { - Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e); - return false; - } - if (existingCredentials.contains(newCredentials)) { - return false; - } - // TODO an alternative approach is to check if we already got an iq result to our - // ICE-restart - // and if that's the case we are seeing an answer. - // This might be more spec compliant but also more error prone potentially - final boolean isOffer = rtpContentMap.emptyCandidates(); - final RtpContentMap restartContentMap; - try { - if (isOffer) { - Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials); - restartContentMap = - existing.modifiedCredentials( - newCredentials, IceUdpTransportInfo.Setup.ACTPASS); - } else { - final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup(); - Log.d( - Config.LOGTAG, - "received confirmation of ICE restart" - + newCredentials - + " peer_setup=" - + setup); - // DTLS setup attribute needs to be rewritten to reflect current peer state - // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM - restartContentMap = existing.modifiedCredentials(newCredentials, setup); - } - if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) { - return isOffer; - } else { - Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break"); - respondWithTieBreak(jinglePacket); - return true; - } - } catch (final Exception exception) { - respondOk(jinglePacket); - final Throwable rootCause = Throwables.getRootCause(exception); - if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) { - // If this happens a termination is already in progress - Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart"); - return true; - } - Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause); - webRTCWrapper.close(); - sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage()); - return true; - } - } - - private IceUdpTransportInfo.Setup getPeerDtlsSetup() { - final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup; - if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) { - throw new IllegalStateException("Invalid peer setup"); - } - return peerSetup; - } - - private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) { - if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) { - throw new IllegalArgumentException("Trying to store invalid peer dtls setup"); - } - this.peerDtlsSetup = setup; - } - - private boolean applyIceRestart( - final JinglePacket jinglePacket, - final RtpContentMap restartContentMap, - final boolean isOffer) - throws ExecutionException, InterruptedException { - final SessionDescription sessionDescription = SessionDescription.of(restartContentMap); - final org.webrtc.SessionDescription.Type type = - isOffer - ? org.webrtc.SessionDescription.Type.OFFER - : org.webrtc.SessionDescription.Type.ANSWER; - org.webrtc.SessionDescription sdp = - new org.webrtc.SessionDescription(type, sessionDescription.toString()); - if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) { - if (isInitiator()) { - // We ignore the offer and respond with tie-break. This will clause the responder - // not to apply the content map - return false; - } - } - webRTCWrapper.setRemoteDescription(sdp).get(); - setRemoteContentMap(restartContentMap); - if (isOffer) { - webRTCWrapper.setIsReadyToReceiveIceCandidates(false); - final SessionDescription localSessionDescription = setLocalSessionDescription(); - setLocalContentMap(RtpContentMap.of(localSessionDescription)); - // We need to respond OK before sending any candidates - respondOk(jinglePacket); - webRTCWrapper.setIsReadyToReceiveIceCandidates(true); - } else { - storePeerDtlsSetup(restartContentMap.getDtlsSetup()); - } - return true; - } - - private void processCandidates( - final Set> contents) { - for (final Map.Entry content : contents) { - processCandidate(content); - } - } - - private void processCandidate( - final Map.Entry content) { - final RtpContentMap rtpContentMap = getRemoteContentMap(); - final List indices = toIdentificationTags(rtpContentMap); - final String sdpMid = content.getKey(); // aka content name - final IceUdpTransportInfo transport = content.getValue().transport; - final IceUdpTransportInfo.Credentials credentials = transport.getCredentials(); - - // TODO check that credentials remained the same - - for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) { - final String sdp; - try { - sdp = candidate.toSdpAttribute(credentials.ufrag); - } catch (final IllegalArgumentException e) { - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": ignoring invalid ICE candidate " - + e.getMessage()); - continue; - } - final int mLineIndex = indices.indexOf(sdpMid); - if (mLineIndex < 0) { - Log.w( - Config.LOGTAG, - "mLineIndex not found for " + sdpMid + ". available indices " + indices); - } - final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp); - Log.d(Config.LOGTAG, "received candidate: " + iceCandidate); - this.webRTCWrapper.addIceCandidate(iceCandidate); - } - } - - private RtpContentMap getRemoteContentMap() { - return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap; - } - - private List toIdentificationTags(final RtpContentMap rtpContentMap) { + private void processCandidates(final Set> contents) { + final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap; final Group originalGroup = rtpContentMap.group; final List identificationTags = originalGroup == null @@ -516,7 +335,30 @@ public class JingleRtpConnection extends AbstractJingleConnection id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices"); } - return identificationTags; + processCandidates(identificationTags, contents); + } + + private void processCandidates(final List indices, final Set> contents) { + for (final Map.Entry content : contents) { + final String ufrag = content.getValue().transport.getAttribute("ufrag"); + for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) { + final String sdp; + try { + sdp = candidate.toSdpAttribute(ufrag); + } catch (IllegalArgumentException e) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage()); + continue; + } + final String sdpMid = content.getKey(); + final int mLineIndex = indices.indexOf(sdpMid); + if (mLineIndex < 0) { + Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices); + } + final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp); + Log.d(Config.LOGTAG, "received candidate: " + iceCandidate); + this.webRTCWrapper.addIceCandidate(iceCandidate); + } + } } private ListenableFuture receiveRtpContentMap( @@ -594,7 +436,7 @@ public class JingleRtpConnection extends AbstractJingleConnection final JinglePacket jinglePacket, final RtpContentMap contentMap) { try { contentMap.requireContentDescriptions(); - contentMap.requireDTLSFingerprint(true); + contentMap.requireDTLSFingerprint(); } catch (final RuntimeException e) { Log.d( Config.LOGTAG, @@ -626,7 +468,11 @@ public class JingleRtpConnection extends AbstractJingleConnection } if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) { respondOk(jinglePacket); - pendingIceCandidates.addAll(contentMap.contents.entrySet()); + + final Set> candidates = contentMap.contents.entrySet(); + if (candidates.size() > 0) { + pendingIceCandidates.push(candidates); + } if (target == State.SESSION_INITIALIZED_PRE_APPROVED) { Log.d( Config.LOGTAG, @@ -728,7 +574,6 @@ public class JingleRtpConnection extends AbstractJingleConnection private void receiveSessionAccept(final RtpContentMap contentMap) { this.responderRtpContentMap = contentMap; - this.storePeerDtlsSetup(contentMap.getDtlsSetup()); final SessionDescription sessionDescription; try { sessionDescription = SessionDescription.of(contentMap); @@ -754,11 +599,11 @@ public class JingleRtpConnection extends AbstractJingleConnection + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e)); webRTCWrapper.close(); - sendSessionTerminate( - Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage()); + sendSessionTerminate(Reason.FAILED_APPLICATION); return; } - processCandidates(contentMap.contents.entrySet()); + final List identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags(); + processCandidates(identificationTags, contentMap.contents.entrySet()); } private void sendSessionAccept() { @@ -811,8 +656,7 @@ public class JingleRtpConnection extends AbstractJingleConnection try { this.webRTCWrapper.setRemoteDescription(sdp).get(); addIceCandidatesFromBlackLog(); - org.webrtc.SessionDescription webRTCSessionDescription = - this.webRTCWrapper.setLocalDescription().get(); + org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get(); prepareSessionAccept(webRTCSessionDescription); } catch (final Exception e) { failureToAcceptSession(e); @@ -823,19 +667,15 @@ public class JingleRtpConnection extends AbstractJingleConnection if (isTerminated()) { return; } - final Throwable rootCause = Throwables.getRootCause(throwable); - Log.d(Config.LOGTAG, "unable to send session accept", rootCause); + Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(throwable)); webRTCWrapper.close(); - sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage()); + sendSessionTerminate(Reason.ofThrowable(throwable)); } private void addIceCandidatesFromBlackLog() { - Map.Entry foo; - while ((foo = this.pendingIceCandidates.poll()) != null) { - processCandidate(foo); - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() + ": added candidate from back log"); + while (!this.pendingIceCandidates.isEmpty()) { + processCandidates(this.pendingIceCandidates.poll()); + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log"); } } @@ -845,16 +685,12 @@ public class JingleRtpConnection extends AbstractJingleConnection SessionDescription.parse(webRTCSessionDescription.description); final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription); this.responderRtpContentMap = respondingRtpContentMap; - storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip()); - webRTCWrapper.setIsReadyToReceiveIceCandidates(true); - final ListenableFuture outgoingContentMapFuture = - prepareOutgoingContentMap(respondingRtpContentMap); - Futures.addCallback( - outgoingContentMapFuture, + final ListenableFuture outgoingContentMapFuture = prepareOutgoingContentMap(respondingRtpContentMap); + Futures.addCallback(outgoingContentMapFuture, new FutureCallback() { @Override public void onSuccess(final RtpContentMap outgoingContentMap) { - sendSessionAccept(outgoingContentMap); + sendSessionAccept(outgoingContentMap, webRTCSessionDescription); } @Override @@ -865,7 +701,7 @@ public class JingleRtpConnection extends AbstractJingleConnection MoreExecutors.directExecutor()); } - private void sendSessionAccept(final RtpContentMap rtpContentMap) { + private void sendSessionAccept(final RtpContentMap rtpContentMap, final org.webrtc.SessionDescription webRTCSessionDescription) { if (isTerminated()) { Log.w( Config.LOGTAG, @@ -877,6 +713,11 @@ public class JingleRtpConnection extends AbstractJingleConnection final JinglePacket sessionAccept = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId); send(sessionAccept); + try { + webRTCWrapper.setLocalDescription(webRTCSessionDescription).get(); + } catch (Exception e) { + failureToAcceptSession(e); + } } private ListenableFuture prepareOutgoingContentMap( @@ -1115,7 +956,6 @@ public class JingleRtpConnection extends AbstractJingleConnection rejectCallFromSessionInitiate(); break; } - xmppConnectionService.getNotificationService().pushMissedCallNow(message); } private void cancelRingingTimeout() { @@ -1193,15 +1033,7 @@ public class JingleRtpConnection extends AbstractJingleConnection this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED; if (transition(target)) { xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); - xmppConnectionService.getNotificationService().pushMissedCallNow(message); - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": session with " - + id.with - + " has been retracted (serverMsgId=" - + serverMsgId - + ")"); + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")"); if (serverMsgId != null) { this.message.setServerMsgId(serverMsgId); } @@ -1256,12 +1088,9 @@ public class JingleRtpConnection extends AbstractJingleConnection return; } try { - org.webrtc.SessionDescription webRTCSessionDescription = - this.webRTCWrapper.setLocalDescription().get(); + org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get(); prepareSessionInitiate(webRTCSessionDescription, targetState); } catch (final Exception e) { - // TODO sending the error text is worthwhile as well. Especially for FailureToSet - // exceptions failureToInitiateSession(e, targetState); } } @@ -1296,16 +1125,12 @@ public class JingleRtpConnection extends AbstractJingleConnection SessionDescription.parse(webRTCSessionDescription.description); final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription); this.initiatorRtpContentMap = rtpContentMap; - this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true); - final ListenableFuture outgoingContentMapFuture = - encryptSessionInitiate(rtpContentMap); - Futures.addCallback( - outgoingContentMapFuture, - new FutureCallback() { - @Override - public void onSuccess(final RtpContentMap outgoingContentMap) { - sendSessionInitiate(outgoingContentMap, targetState); - } + final ListenableFuture outgoingContentMapFuture = encryptSessionInitiate(rtpContentMap); + Futures.addCallback(outgoingContentMapFuture, new FutureCallback() { + @Override + public void onSuccess(final RtpContentMap outgoingContentMap) { + sendSessionInitiate(outgoingContentMap, webRTCSessionDescription, targetState); + } @Override public void onFailure(@NonNull final Throwable throwable) { @@ -1315,7 +1140,7 @@ public class JingleRtpConnection extends AbstractJingleConnection MoreExecutors.directExecutor()); } - private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) { + private void sendSessionInitiate(final RtpContentMap rtpContentMap, final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) { if (isTerminated()) { Log.w( Config.LOGTAG, @@ -1327,6 +1152,11 @@ public class JingleRtpConnection extends AbstractJingleConnection final JinglePacket sessionInitiate = rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId); send(sessionInitiate); + try { + this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get(); + } catch (Exception e) { + failureToInitiateSession(e, targetState); + } } private ListenableFuture encryptSessionInitiate( @@ -1415,65 +1245,36 @@ public class JingleRtpConnection extends AbstractJingleConnection private synchronized void handleIqResponse(final Account account, final IqPacket response) { if (response.getType() == IqPacket.TYPE.ERROR) { - handleIqErrorResponse(response); - return; - } - if (response.getType() == IqPacket.TYPE.TIMEOUT) { - handleIqTimeoutResponse(response); - } - } - - private void handleIqErrorResponse(final IqPacket response) { - Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR); - final String errorCondition = response.getErrorCondition(); - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": received IQ-error from " - + response.getFrom() - + " in RTP session. " - + errorCondition); - if (isTerminated()) { - Log.i( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": ignoring error because session was already terminated"); - return; - } - this.webRTCWrapper.close(); - final State target; - if (Arrays.asList( - "service-unavailable", - "recipient-unavailable", - "remote-server-not-found", - "remote-server-timeout") - .contains(errorCondition)) { - target = State.TERMINATED_CONNECTIVITY_ERROR; - } else { - target = State.TERMINATED_APPLICATION_FAILURE; - } - transitionOrThrow(target); - this.finish(); - } - - private void handleIqTimeoutResponse(final IqPacket response) { - Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT); - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": received IQ timeout in RTP session with " - + id.with - + ". terminating with connectivity error"); - if (isTerminated()) { - Log.i( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": ignoring error because session was already terminated"); - return; + final String errorCondition = response.getErrorCondition(); + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition); + if (isTerminated()) { + Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated"); + return; + } + this.webRTCWrapper.close(); + final State target; + if (Arrays.asList( + "service-unavailable", + "recipient-unavailable", + "remote-server-not-found", + "remote-server-timeout" + ).contains(errorCondition)) { + target = State.TERMINATED_CONNECTIVITY_ERROR; + } else { + target = State.TERMINATED_APPLICATION_FAILURE; + } + transitionOrThrow(target); + this.finish(); + } else if (response.getType() == IqPacket.TYPE.TIMEOUT) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error"); + if (isTerminated()) { + Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated"); + return; + } + this.webRTCWrapper.close(); + transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR); + this.finish(); } - this.webRTCWrapper.close(); - transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR); - this.finish(); } private void terminateWithOutOfOrder(final JinglePacket jinglePacket) { @@ -1486,21 +1287,8 @@ public class JingleRtpConnection extends AbstractJingleConnection this.finish(); } - private void respondWithTieBreak(final JinglePacket jinglePacket) { - respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel"); - } - private void respondWithOutOfOrder(final JinglePacket jinglePacket) { - respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait"); - } - - void respondWithJingleError( - final IqPacket original, - String jingleCondition, - String condition, - String conditionType) { - jingleConnectionManager.respondWithJingleError( - id.account, original, jingleCondition, condition, conditionType); + jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait"); } private void respondOk(final JinglePacket jinglePacket) { @@ -1531,7 +1319,24 @@ public class JingleRtpConnection extends AbstractJingleConnection return RtpEndUserState.CONNECTING; } case SESSION_ACCEPTED: - return getPeerConnectionStateAsEndUserState(); + //TODO refactor this out into separate method (that uses switch for better readability) + final PeerConnection.PeerConnectionState state; + try { + state = webRTCWrapper.getState(); + } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { + //We usually close the WebRTCWrapper *before* transitioning so we might still + //be in SESSION_ACCEPTED even though the peerConnection has been torn down + return RtpEndUserState.ENDING_CALL; + } + if (state == PeerConnection.PeerConnectionState.CONNECTED) { + return RtpEndUserState.CONNECTED; + } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) { + return RtpEndUserState.CONNECTING; + } else if (state == PeerConnection.PeerConnectionState.CLOSED) { + return RtpEndUserState.ENDING_CALL; + } else { + return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR; + } case REJECTED: case REJECTED_RACED: case TERMINATED_DECLINED_OR_BUSY: @@ -1564,30 +1369,6 @@ public class JingleRtpConnection extends AbstractJingleConnection String.format("%s has no equivalent EndUserState", this.state)); } - private RtpEndUserState getPeerConnectionStateAsEndUserState() { - final PeerConnection.PeerConnectionState state; - try { - state = webRTCWrapper.getState(); - } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { - // We usually close the WebRTCWrapper *before* transitioning so we might still - // be in SESSION_ACCEPTED even though the peerConnection has been torn down - return RtpEndUserState.ENDING_CALL; - } - switch (state) { - case CONNECTED: - return RtpEndUserState.CONNECTED; - case NEW: - case CONNECTING: - return RtpEndUserState.CONNECTING; - case CLOSED: - return RtpEndUserState.ENDING_CALL; - default: - return zeroDuration() - ? RtpEndUserState.CONNECTIVITY_ERROR - : RtpEndUserState.RECONNECTING; - } - } - public Set getMedia() { final State current = getState(); if (current == State.NULL) { @@ -1852,133 +1633,40 @@ public class JingleRtpConnection extends AbstractJingleConnection @Override public void onIceCandidate(final IceCandidate iceCandidate) { - final RtpContentMap rtpContentMap = - isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; - final IceUdpTransportInfo.Credentials credentials; - try { - credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid); - } catch (final IllegalArgumentException e) { - Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e); - return; - } - final String uFrag = credentials.ufrag; - final IceUdpTransportInfo.Candidate candidate = - IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag); - if (candidate == null) { - Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate); - return; - } - Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate); + final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp); + Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString()); sendTransportInfo(iceCandidate.sdpMid, candidate); } @Override public void onConnectionChange(final PeerConnection.PeerConnectionState newState) { - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState); - this.stateHistory.add(newState); - if (newState == PeerConnection.PeerConnectionState.CONNECTED) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState); + if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) { + this.rtpConnectionStarted = SystemClock.elapsedRealtime(); this.sessionDuration.start(); - updateOngoingCallNotification(); - } else if (this.sessionDuration.isRunning()) { - this.sessionDuration.stop(); - updateOngoingCallNotification(); } - - final boolean neverConnected = - !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED); - - if (newState == PeerConnection.PeerConnectionState.FAILED) { - if (neverConnected) { - if (isTerminated()) { - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": not sending session-terminate after connectivity error because session is already in state " - + this.state); - return; - } - webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection); + if (newState == PeerConnection.PeerConnectionState.CLOSED && this.rtpConnectionEnded == 0) { + this.rtpConnectionEnded = SystemClock.elapsedRealtime(); + if (this.sessionDuration.isRunning()) this.sessionDuration.stop(); + } + //TODO 'failed' means we need to restart ICE + // + //TODO 'disconnected' can probably be ignored as "This is a less stringent test than failed + // and may trigger intermittently and resolve just as spontaneously on less reliable networks, + // or during temporary disconnections. When the problem resolves, the connection may return + // to the connected state." + // Obviously the UI needs to reflect this new state with a 'reconnecting' display or something + if (Arrays.asList(PeerConnection.PeerConnectionState.FAILED, PeerConnection.PeerConnectionState.DISCONNECTED).contains(newState)) { + if (isTerminated()) { + Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state); return; - } else { - webRTCWrapper.restartIce(); } - } - updateEndUserState(); - } - - @Override - public void onRenegotiationNeeded() { - this.webRTCWrapper.execute(this::initiateIceRestart); - } - - private void initiateIceRestart() { - // TODO discover new TURN/STUN credentials - this.stateHistory.clear(); - this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false); - final SessionDescription sessionDescription; - try { - sessionDescription = setLocalSessionDescription(); - } catch (final Exception e) { - final Throwable cause = Throwables.getRootCause(e); - Log.d(Config.LOGTAG, "failed to renegotiate", cause); - sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); - return; - } - final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription); - final RtpContentMap transportInfo = rtpContentMap.transportInfo(); - final JinglePacket jinglePacket = - transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId); - Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket); - jinglePacket.setTo(id.with); - xmppConnectionService.sendIqPacket( - id.account, - jinglePacket, - (account, response) -> { - if (response.getType() == IqPacket.TYPE.RESULT) { - Log.d(Config.LOGTAG, "received success to our ice restart"); - setLocalContentMap(rtpContentMap); - webRTCWrapper.setIsReadyToReceiveIceCandidates(true); - return; - } - if (response.getType() == IqPacket.TYPE.ERROR) { - final Element error = response.findChild("error"); - if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) { - Log.d(Config.LOGTAG, "received tie-break as result of ice restart"); - return; - } - handleIqErrorResponse(response); - } - if (response.getType() == IqPacket.TYPE.TIMEOUT) { - handleIqTimeoutResponse(response); - } - }); - } - - private void setLocalContentMap(final RtpContentMap rtpContentMap) { - if (isInitiator()) { - this.initiatorRtpContentMap = rtpContentMap; - } else { - this.responderRtpContentMap = rtpContentMap; - } - } - - private void setRemoteContentMap(final RtpContentMap rtpContentMap) { - if (isInitiator()) { - this.responderRtpContentMap = rtpContentMap; + new Thread(this::closeWebRTCSessionAfterFailedConnection).start(); } else { - this.initiatorRtpContentMap = rtpContentMap; + updateEndUserState(); } } - private SessionDescription setLocalSessionDescription() - throws ExecutionException, InterruptedException { - final org.webrtc.SessionDescription sessionDescription = - this.webRTCWrapper.setLocalDescription().get(); - return SessionDescription.parse(sessionDescription.description); - } - private void closeWebRTCSessionAfterFailedConnection() { this.webRTCWrapper.close(); synchronized (this) { @@ -1993,6 +1681,14 @@ public class JingleRtpConnection extends AbstractJingleConnection } } + public long getRtpConnectionStarted() { + return this.rtpConnectionStarted; + } + + public long getRtpConnectionEnded() { + return this.rtpConnectionEnded; + } + public boolean zeroDuration() { return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0; } @@ -2049,16 +1745,8 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void updateOngoingCallNotification() { - final State state = this.state; - if (STATES_SHOWING_ONGOING_CALL.contains(state)) { - final boolean reconnecting; - if (state == State.SESSION_ACCEPTED) { - reconnecting = - getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING; - } else { - reconnecting = false; - } - xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting); + if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) { + xmppConnectionService.setOngoingCall(id, getMedia(), false); } else { xmppConnectionService.removeOngoingCall(); } @@ -2182,9 +1870,9 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void writeLogMessage(final State state) { - final long duration = getCallDuration(); - if (state == State.TERMINATED_SUCCESS - || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) { + final long started = this.rtpConnectionStarted; + long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started; + if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) { writeLogMessageSuccess(duration); } else { writeLogMessageMissed(); @@ -2228,6 +1916,7 @@ public class JingleRtpConnection extends AbstractJingleConnection return webRTCWrapper.getRemoteVideoTrack(); } + public EglBase.Context getEglBaseContext() { return webRTCWrapper.getEglBaseContext(); } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java index e95a7e36d..0c40fffd0 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java @@ -1,12 +1,13 @@ package eu.siacs.conversations.xmpp.jingle; +import android.util.Log; + import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -18,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import eu.siacs.conversations.Config; import eu.siacs.conversations.xmpp.jingle.stanzas.Content; import eu.siacs.conversations.xmpp.jingle.stanzas.GenericDescription; import eu.siacs.conversations.xmpp.jingle.stanzas.GenericTransportInfo; @@ -105,10 +107,6 @@ public class RtpContentMap { } void requireDTLSFingerprint() { - requireDTLSFingerprint(false); - } - - void requireDTLSFingerprint(final boolean requireActPass) { if (this.contents.size() == 0) { throw new IllegalStateException("No contents available"); } @@ -123,16 +121,8 @@ public class RtpContentMap { "Use of DTLS-SRTP (XEP-0320) is required for content %s", entry.getKey())); } - final IceUdpTransportInfo.Setup setup = fingerprint.getSetup(); - if (setup == null) { - throw new SecurityException( - String.format( - "Use of DTLS-SRTP (XEP-0320) is required for content %s but missing setup attribute", - entry.getKey())); - } - if (requireActPass && setup != IceUdpTransportInfo.Setup.ACTPASS) { - throw new SecurityException( - "Initiator needs to offer ACTPASS as setup for DTLS-SRTP (XEP-0320)"); + if (Strings.isNullOrEmpty(fingerprint.getSetup())) { + throw new SecurityException(String.format("Use of DTLS-SRTP (XEP-0320) is required for content %s but missing setup attribute", entry.getKey())); } } } @@ -164,84 +154,7 @@ public class RtpContentMap { } final IceUdpTransportInfo newTransportInfo = transportInfo.cloneWrapper(); newTransportInfo.addChild(candidate); - return new RtpContentMap( - null, - ImmutableMap.of(contentName, new DescriptionTransport(null, newTransportInfo))); - } - - RtpContentMap transportInfo() { - return new RtpContentMap( - null, - Maps.transformValues( - contents, - dt -> new DescriptionTransport(null, dt.transport.cloneWrapper()))); - } - - public IceUdpTransportInfo.Credentials getDistinctCredentials() { - final Set allCredentials = getCredentials(); - final IceUdpTransportInfo.Credentials credentials = - Iterables.getFirst(allCredentials, null); - if (allCredentials.size() == 1 && credentials != null) { - return credentials; - } - throw new IllegalStateException("Content map does not have distinct credentials"); - } - - public Set getCredentials() { - final Set credentials = - ImmutableSet.copyOf( - Collections2.transform( - contents.values(), dt -> dt.transport.getCredentials())); - if (credentials.isEmpty()) { - throw new IllegalStateException("Content map does not have any credentials"); - } - return credentials; - } - - public IceUdpTransportInfo.Credentials getCredentials(final String contentName) { - final DescriptionTransport descriptionTransport = this.contents.get(contentName); - if (descriptionTransport == null) { - throw new IllegalArgumentException( - String.format( - "Unable to find transport info for content name %s", contentName)); - } - return descriptionTransport.transport.getCredentials(); - } - - public IceUdpTransportInfo.Setup getDtlsSetup() { - final Set setups = - ImmutableSet.copyOf( - Collections2.transform( - contents.values(), dt -> dt.transport.getFingerprint().getSetup())); - final IceUdpTransportInfo.Setup setup = Iterables.getFirst(setups, null); - if (setups.size() == 1 && setup != null) { - return setup; - } - throw new IllegalStateException("Content map doesn't have distinct DTLS setup"); - } - - public boolean emptyCandidates() { - int count = 0; - for (DescriptionTransport descriptionTransport : contents.values()) { - count += descriptionTransport.transport.getCandidates().size(); - } - return count == 0; - } - - public RtpContentMap modifiedCredentials( - IceUdpTransportInfo.Credentials credentials, final IceUdpTransportInfo.Setup setup) { - final ImmutableMap.Builder contentMapBuilder = - new ImmutableMap.Builder<>(); - for (final Map.Entry content : contents.entrySet()) { - final RtpDescription rtpDescription = content.getValue().description; - IceUdpTransportInfo transportInfo = content.getValue().transport; - final IceUdpTransportInfo modifiedTransportInfo = - transportInfo.modifyCredentials(credentials, setup); - contentMapBuilder.put( - content.getKey(), - new DescriptionTransport(rtpDescription, modifiedTransportInfo)); - } - return new RtpContentMap(this.group, contentMapBuilder.build()); + return new RtpContentMap(null, ImmutableMap.of(contentName, new DescriptionTransport(null, newTransportInfo))); } public static class DescriptionTransport { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java index e113146b1..39031c4a9 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java @@ -156,10 +156,7 @@ public class SessionDescription { final IceUdpTransportInfo.Fingerprint fingerprint = transport.getFingerprint(); if (fingerprint != null) { mediaAttributes.put("fingerprint", fingerprint.getHash() + " " + fingerprint.getContent()); - final IceUdpTransportInfo.Setup setup = fingerprint.getSetup(); - if (setup != null) { - mediaAttributes.put("setup", setup.toString().toLowerCase(Locale.ROOT)); - } + mediaAttributes.put("setup", fingerprint.getSetup()); } final ImmutableList.Builder formatBuilder = new ImmutableList.Builder<>(); for (RtpDescription.PayloadType payloadType : description.getPayloadTypes()) { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index 78c3d0a71..4abd2e170 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.SettableFuture; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; +import org.webrtc.Camera1Enumerator; import org.webrtc.Camera2Enumerator; import org.webrtc.CameraEnumerationAndroid; import org.webrtc.CameraEnumerator; @@ -47,14 +48,9 @@ import org.webrtc.voiceengine.WebRtcAudioEffects; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; import java.util.List; -import java.util.Queue; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -67,8 +63,7 @@ public class WebRTCWrapper { private static final String EXTENDED_LOGGING_TAG = WebRTCWrapper.class.getSimpleName(); - private final ExecutorService executorService = Executors.newSingleThreadExecutor(); - + //we should probably keep this in sync with: https://github.com/signalapp/Signal-Android/blob/master/app/src/main/java/org/thoughtcrime/securesms/ApplicationContext.java#L296 private static final Set HARDWARE_AEC_BLACKLIST = new ImmutableSet.Builder() .add("Pixel") .add("Pixel XL") @@ -110,8 +105,6 @@ public class WebRTCWrapper { private static final int CAPTURING_MAX_FRAME_RATE = 30; private final EventCallback eventCallback; - private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false); - private final Queue iceCandidates = new LinkedList<>(); private final AppRTCAudioManager.AudioManagerEvents audioManagerEvents = new AppRTCAudioManager.AudioManagerEvents() { @Override public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set availableAudioDevices) { @@ -131,13 +124,13 @@ public class WebRTCWrapper { } @Override - public void onConnectionChange(final PeerConnection.PeerConnectionState newState) { + public void onConnectionChange(PeerConnection.PeerConnectionState newState) { eventCallback.onConnectionChange(newState); } @Override public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { - Log.d(EXTENDED_LOGGING_TAG, "onIceConnectionChange(" + iceConnectionState + ")"); + } @Override @@ -158,11 +151,7 @@ public class WebRTCWrapper { @Override public void onIceCandidate(IceCandidate iceCandidate) { - if (readyToReceivedIceCandidates.get()) { - eventCallback.onIceCandidate(iceCandidate); - } else { - iceCandidates.add(iceCandidate); - } + eventCallback.onIceCandidate(iceCandidate); } @Override @@ -187,11 +176,7 @@ public class WebRTCWrapper { @Override public void onRenegotiationNeeded() { - Log.d(EXTENDED_LOGGING_TAG, "onRenegotiationNeeded()"); - final PeerConnection.PeerConnectionState currentState = peerConnection == null ? null : peerConnection.connectionState(); - if (currentState != null && currentState != PeerConnection.PeerConnectionState.NEW) { - eventCallback.onRenegotiationNeeded(); - } + } @Override @@ -294,7 +279,11 @@ public class WebRTCWrapper { .createPeerConnectionFactory(); - final PeerConnection.RTCConfiguration rtcConfig = buildConfiguration(iceServers); + final PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers); + rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; //XEP-0176 doesn't support tcp + rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; + rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; + rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE; final PeerConnection peerConnection = peerConnectionFactory.createPeerConnection(rtcConfig, peerConnectionObserver); if (peerConnection == null) { throw new InitializationException("Unable to create PeerConnection"); @@ -328,31 +317,6 @@ public class WebRTCWrapper { this.peerConnection = peerConnection; } - private static PeerConnection.RTCConfiguration buildConfiguration(final List iceServers) { - final PeerConnection.RTCConfiguration rtcConfig = new PeerConnection.RTCConfiguration(iceServers); - rtcConfig.tcpCandidatePolicy = PeerConnection.TcpCandidatePolicy.DISABLED; //XEP-0176 doesn't support tcp - rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; - rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; - rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE; - rtcConfig.enableImplicitRollback = true; - return rtcConfig; - } - - void reconfigurePeerConnection(final List iceServers) { - requirePeerConnection().setConfiguration(buildConfiguration(iceServers)); - } - - void restartIce() { - executorService.execute(() -> requirePeerConnection().restartIce()); - } - - public void setIsReadyToReceiveIceCandidates(final boolean ready) { - readyToReceivedIceCandidates.set(ready); - while (ready && iceCandidates.peek() != null) { - eventCallback.onIceCandidate(iceCandidates.poll()); - } - } - synchronized void close() { final PeerConnection peerConnection = this.peerConnection; final CapturerChoice capturerChoice = this.capturerChoice; @@ -467,36 +431,70 @@ public class WebRTCWrapper { videoTrack.setEnabled(enabled); } - synchronized ListenableFuture setLocalDescription() { + ListenableFuture createOffer() { return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> { final SettableFuture future = SettableFuture.create(); - peerConnection.setLocalDescription(new SetSdpObserver() { + peerConnection.createOffer(new CreateSdpObserver() { @Override - public void onSetSuccess() { - final SessionDescription description = peerConnection.getLocalDescription(); - Log.d(EXTENDED_LOGGING_TAG, "set local description:"); - logDescription(description); - future.set(description); + public void onCreateSuccess(SessionDescription sessionDescription) { + future.set(sessionDescription); } @Override - public void onSetFailure(final String message) { - future.setException(new FailureToSetDescriptionException(message)); + public void onCreateFailure(String s) { + future.setException(new IllegalStateException("Unable to create offer: " + s)); } - }); + }, new MediaConstraints()); return future; }, MoreExecutors.directExecutor()); } - private static void logDescription(final SessionDescription sessionDescription) { + ListenableFuture createAnswer() { + return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> { + final SettableFuture future = SettableFuture.create(); + peerConnection.createAnswer(new CreateSdpObserver() { + @Override + public void onCreateSuccess(SessionDescription sessionDescription) { + future.set(sessionDescription); + } + + @Override + public void onCreateFailure(String s) { + future.setException(new IllegalStateException("Unable to create answer: " + s)); + } + }, new MediaConstraints()); + return future; + }, MoreExecutors.directExecutor()); + } + + ListenableFuture setLocalDescription(final SessionDescription sessionDescription) { + Log.d(EXTENDED_LOGGING_TAG, "setting local description:"); for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) { Log.d(EXTENDED_LOGGING_TAG, line); } + return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> { + final SettableFuture future = SettableFuture.create(); + peerConnection.setLocalDescription(new SetSdpObserver() { + @Override + public void onSetSuccess() { + future.set(null); + } + + @Override + public void onSetFailure(final String s) { + future.setException(new IllegalArgumentException("unable to set local session description: " + s)); + + } + }, sessionDescription); + return future; + }, MoreExecutors.directExecutor()); } - synchronized ListenableFuture setRemoteDescription(final SessionDescription sessionDescription) { + ListenableFuture setRemoteDescription(final SessionDescription sessionDescription) { Log.d(EXTENDED_LOGGING_TAG, "setting remote description:"); - logDescription(sessionDescription); + for (final String line : sessionDescription.description.split(eu.siacs.conversations.xmpp.jingle.SessionDescription.LINE_DIVIDER)) { + Log.d(EXTENDED_LOGGING_TAG, line); + } return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> { final SettableFuture future = SettableFuture.create(); peerConnection.setRemoteDescription(new SetSdpObserver() { @@ -506,8 +504,9 @@ public class WebRTCWrapper { } @Override - public void onSetFailure(final String message) { - future.setException(new FailureToSetDescriptionException(message)); + public void onSetFailure(String s) { + future.setException(new IllegalArgumentException("unable to set remote session description: " + s)); + } }, sessionDescription); return future; @@ -518,20 +517,12 @@ public class WebRTCWrapper { private ListenableFuture getPeerConnectionFuture() { final PeerConnection peerConnection = this.peerConnection; if (peerConnection == null) { - return Futures.immediateFailedFuture(new PeerConnectionNotInitialized()); + return Futures.immediateFailedFuture(new IllegalStateException("initialize PeerConnection first")); } else { return Futures.immediateFuture(peerConnection); } } - private PeerConnection requirePeerConnection() { - final PeerConnection peerConnection = this.peerConnection; - if (peerConnection == null) { - throw new PeerConnectionNotInitialized(); - } - return peerConnection; - } - public boolean applyDtmfTone(String tone) { if (toneManager == null || peerConnection.getSenders().isEmpty()) { return false; @@ -545,8 +536,16 @@ public class WebRTCWrapper { requirePeerConnection().addIceCandidate(iceCandidate); } + private CameraEnumerator getCameraEnumerator() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { + return new Camera2Enumerator(requireContext()); + } else { + return new Camera1Enumerator(); + } + } + private Optional getVideoCapturer() { - final CameraEnumerator enumerator = new Camera2Enumerator(requireContext()); + final CameraEnumerator enumerator = getCameraEnumerator(); final Set deviceNames = ImmutableSet.copyOf(enumerator.getDeviceNames()); for (final String deviceName : deviceNames) { if (isFrontFacing(enumerator, deviceName)) { @@ -565,15 +564,10 @@ public class WebRTCWrapper { } } - PeerConnection.PeerConnectionState getState() { + public PeerConnection.PeerConnectionState getState() { return requirePeerConnection().connectionState(); } - public PeerConnection.SignalingState getSignalingState() { - return requirePeerConnection().signalingState(); - } - - EglBase.Context getEglBaseContext() { return this.eglBase.getEglBaseContext(); } @@ -586,6 +580,14 @@ public class WebRTCWrapper { return Optional.fromNullable(this.remoteVideoTrack); } + private PeerConnection requirePeerConnection() { + final PeerConnection peerConnection = this.peerConnection; + if (peerConnection == null) { + throw new PeerConnectionNotInitialized(); + } + return peerConnection; + } + private Context requireContext() { final Context context = this.context; if (context == null) { @@ -598,18 +600,12 @@ public class WebRTCWrapper { return appRTCAudioManager; } - void execute(final Runnable command) { - executorService.execute(command); - } - public interface EventCallback { void onIceCandidate(IceCandidate iceCandidate); void onConnectionChange(PeerConnection.PeerConnectionState newState); void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set availableAudioDevices); - - void onRenegotiationNeeded(); } private static abstract class SetSdpObserver implements SdpObserver { @@ -660,12 +656,6 @@ public class WebRTCWrapper { } - private static class FailureToSetDescriptionException extends IllegalArgumentException { - public FailureToSetDescriptionException(String message) { - super(message); - } - } - private static class CapturerChoice { private final CameraVideoCapturer cameraVideoCapturer; private final CameraEnumerationAndroid.CaptureFormat captureFormat; diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java index ee8d12b70..b70e3369b 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java @@ -3,8 +3,6 @@ package eu.siacs.conversations.xmpp.jingle.stanzas; import androidx.annotation.NonNull; import com.google.common.base.Joiner; -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ArrayListMultimap; @@ -12,8 +10,6 @@ import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; -import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.Hashtable; import java.util.LinkedHashMap; @@ -64,12 +60,6 @@ public class IceUdpTransportInfo extends GenericTransportInfo { return fingerprint == null ? null : Fingerprint.upgrade(fingerprint); } - public Credentials getCredentials() { - final String ufrag = this.getAttribute("ufrag"); - final String password = this.getAttribute("pwd"); - return new Credentials(ufrag, password); - } - public List getCandidates() { final ImmutableList.Builder builder = new ImmutableList.Builder<>(); for (final Element child : getChildren()) { @@ -86,54 +76,6 @@ public class IceUdpTransportInfo extends GenericTransportInfo { return transportInfo; } - public IceUdpTransportInfo modifyCredentials(final Credentials credentials, final Setup setup) { - final IceUdpTransportInfo transportInfo = new IceUdpTransportInfo(); - transportInfo.setAttribute("ufrag", credentials.ufrag); - transportInfo.setAttribute("pwd", credentials.password); - for (final Element child : getChildren()) { - if (child.getName().equals("fingerprint") && Namespace.JINGLE_APPS_DTLS.equals(child.getNamespace())) { - final Fingerprint fingerprint = new Fingerprint(); - fingerprint.setAttributes(new Hashtable<>(child.getAttributes())); - fingerprint.setContent(child.getContent()); - fingerprint.setAttribute("setup", setup.toString().toLowerCase(Locale.ROOT)); - transportInfo.addChild(fingerprint); - } - } - return transportInfo; - } - - public static class Credentials { - public final String ufrag; - public final String password; - - public Credentials(String ufrag, String password) { - this.ufrag = ufrag; - this.password = password; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Credentials that = (Credentials) o; - return Objects.equal(ufrag, that.ufrag) && Objects.equal(password, that.password); - } - - @Override - public int hashCode() { - return Objects.hashCode(ufrag, password); - } - - @Override - @NonNull - public String toString() { - return MoreObjects.toStringHelper(this) - .add("ufrag", ufrag) - .add("password", password) - .toString(); - } - } - public static class Candidate extends Element { private Candidate() { @@ -149,7 +91,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo { } // https://tools.ietf.org/html/draft-ietf-mmusic-ice-sip-sdp-39#section-5.1 - public static Candidate fromSdpAttribute(final String attribute, String currentUfrag) { + public static Candidate fromSdpAttribute(final String attribute) { final String[] pair = attribute.split(":", 2); if (pair.length == 2 && "candidate".equals(pair[0])) { final String[] segments = pair[1].split(" "); @@ -165,10 +107,6 @@ public class IceUdpTransportInfo extends GenericTransportInfo { for (int i = 6; i < segments.length - 1; i = i + 2) { additional.put(segments[i], segments[i + 1]); } - final String ufrag = additional.get("ufrag"); - if (ufrag != null && !ufrag.equals(currentUfrag)) { - return null; - } final Candidate candidate = new Candidate(); candidate.setAttribute("component", component); candidate.setAttribute("foundation", foundation); @@ -349,31 +287,8 @@ public class IceUdpTransportInfo extends GenericTransportInfo { return this.getAttribute("hash"); } - public Setup getSetup() { - final String setup = this.getAttribute("setup"); - return setup == null ? null : Setup.of(setup); - } - } - - public enum Setup { - ACTPASS, PASSIVE, ACTIVE; - - public static Setup of(String setup) { - try { - return valueOf(setup.toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { - return null; - } - } - - public Setup flip() { - if (this == PASSIVE) { - return ACTIVE; - } - if (this == ACTIVE) { - return PASSIVE; - } - throw new IllegalStateException(this.name()+" can not be flipped"); + public String getSetup() { + return this.getAttribute("setup"); } } } -- 2.34.7