Flutter WebRTC 音频但没有视频

Question

所以我正在使用 flutter、flutterWeb 和 WebRTC 包构建一个视频通话应用程序。

我有一个 spring 启动服务器位于中间，用于在两个客户端之间传递消息。

双方都显示本地视频，但都不显示远程。音频确实有效。我有一些讨厌的反馈循环。使用耳机进行测试表明音频确实有效。

我的信号码

typedef void StreamStateCallback(MediaStream stream);

class CallingService {
  String sendToUserId;
  String currentUserId;
  final String authToken;
  final StompClient _client;
  final StreamStateCallback onAddRemoteStream;
  final StreamStateCallback onRemoveRemoteStream;
  final StreamStateCallback onAddLocalStream;
  RTCPeerConnection _peerConnection;
  List<RTCIceCandidate> _remoteCandidates = [];
  String destination;
  var hasOffer = false;
  var isNegotiating = false;
  MediaStream _localStream;

  final Map<String, dynamic> _constraints = {
    'mandatory': {
      'OfferToReceiveAudio': true,
      'OfferToReceiveVideo': true,
    },
    'optional': [],
  };

  CallingService(
      this._client,
      this.sendToUserId,
      this.currentUserId,
      this.authToken,
      this.onAddRemoteStream,
      this.onRemoveRemoteStream,
      this.onAddLocalStream) {
    destination = '/app/start-call/$sendToUserId';
    print("destination $destination");
    _client.subscribe(
        destination: destination,
        headers: {'Authorization': "$authToken"},
        callback: (StompFrame frame) => processMessage(jsonDecode(frame.body)));
  }

  Future<void> startCall() async {
    await processRemoteStream();
    RTCSessionDescription description =
        await _peerConnection.createOffer(_constraints);
    await _peerConnection.setLocalDescription(description);
    var message = RtcMessage(RtcMessageType.OFFER, currentUserId, {
      'description': {'sdp': description.sdp, 'type': description.type},
    });
    sendMessage(message);
  }

  Future<void> processMessage(Map<String, dynamic> messageJson) async {
    var message = RtcMessage.fromJson(messageJson);
    if (message.from == currentUserId) {
      return;
    }
    print("processing ${message.messageType.toString()}");
    switch (message.messageType) {
      case RtcMessageType.BYE:
        // TODO: Handle this case.
        break;
      case RtcMessageType.LEAVE:
        // TODO: Handle this case.
        break;
      case RtcMessageType.CANDIDATE:
        await processCandidate(message);
        break;
      case RtcMessageType.ANSWER:
        await processAnswer(message);
        break;
      case RtcMessageType.OFFER:
        await processOffer(message);
        break;
    }
  }

  Future<void> processCandidate(RtcMessage candidate) async {
    Map<String, dynamic> map = candidate.data['candidate'];
    var rtcCandidate = RTCIceCandidate(
      map['candidate'],
      map['sdpMid'],
      map['sdpMLineIndex'],
    );
    if (_peerConnection != null) {
      _peerConnection.addCandidate(rtcCandidate);
    } else {
      _remoteCandidates.add(rtcCandidate);
    }
  }

  Future<void> processAnswer(RtcMessage answer) async {
    if (isNegotiating) {
      return;
    }
    isNegotiating = true;
    var description = answer.data['description'];
    if (_peerConnection == null) {
      return;
    }
    await _peerConnection.setRemoteDescription(
        RTCSessionDescription(description['sdp'], description['type']));
  }

  Future<void> processOffer(RtcMessage offer) async {
    await processRemoteStream();
    var description = offer.data['description'];
    await _peerConnection.setRemoteDescription(
        new RTCSessionDescription(description['sdp'], description['type']));
    var answerDescription = await _peerConnection.createAnswer(_constraints);
    await _peerConnection.setLocalDescription(answerDescription);
    var answerMessage = RtcMessage(RtcMessageType.ANSWER, currentUserId, {
      'description': {
        'sdp': answerDescription.sdp,
        'type': answerDescription.type
      },
    });
    sendMessage(answerMessage);
    if (_remoteCandidates.isNotEmpty) {
      _remoteCandidates
          .forEach((candidate) => _peerConnection.addCandidate(candidate));
      _remoteCandidates.clear();
    }
  }

  Future<void> processRemoteStream() async {
    _localStream = await createStream();
    _peerConnection = await createPeerConnection(_iceServers, _config);
    _peerConnection.addStream(_localStream);
    _peerConnection.onSignalingState = (state) {
      //isNegotiating = state != RTCSignalingState.RTCSignalingStateStable;
    };

    _peerConnection.onAddStream = (MediaStream stream) {
      this.onAddRemoteStream(stream);
    };
    _peerConnection.onRemoveStream =
        (MediaStream stream) => this.onRemoveRemoteStream(stream);
    _peerConnection.onIceCandidate = (RTCIceCandidate candidate) {
      var data = {
        'candidate': {
          'sdpMLineIndex': candidate.sdpMlineIndex,
          'sdpMid': candidate.sdpMid,
          'candidate': candidate.candidate,
        },
      };
      var message = RtcMessage(RtcMessageType.CANDIDATE, currentUserId, data);
      sendMessage(message);
    };
  }

  void sendMessage(RtcMessage message) {
    _client.send(
        destination: destination,
        headers: {'Authorization': "$authToken"},
        body: jsonEncode(message.toJson()));
  }

  Map<String, dynamic> _iceServers = {
    'iceServers': [
      {'urls': 'stun:stun.l.google.com:19302'},
      /*
       * turn server configuration example.
      {
        'url': 'turn:123.45.67.89:3478',
        'username': 'change_to_real_user',
        'credential': 'change_to_real_secret'
      },
       */
    ]
  };

  final Map<String, dynamic> _config = {
    'mandatory': {},
    'optional': [
      {'DtlsSrtpKeyAgreement': true},
    ],
  };

  Future<MediaStream> createStream() async {
    final Map<String, dynamic> mediaConstraints = {
      'audio': true,
      'video': {
        'mandatory': {
          'minWidth': '640',
          'minHeight': '480',
          'minFrameRate': '30',
        },
        'facingMode': 'user',
        'optional': [],
      }
    };

    MediaStream stream = await navigator.getUserMedia(mediaConstraints);
    if (this.onAddLocalStream != null) {
      this.onAddLocalStream(stream);
    }
    return stream;
  }
}

这是我的小部件

class _CallScreenState extends State<CallScreen> {
  StompClient _client;
  CallingService _callingService;
  RTCVideoRenderer _localRenderer = new RTCVideoRenderer();
  RTCVideoRenderer _remoteRenderer = new RTCVideoRenderer();
  final UserService userService = GetIt.instance.get<UserService>();

  void onConnectCallback(StompClient client, StompFrame connectFrame) async {
    var currentUser = await userService.getCurrentUser();
    _callingService = CallingService(
        _client,
        widget.intent.toUserId.toString(),
        currentUser.id.toString(),
        widget.intent.authToken,
        onAddRemoteStream,
        onRemoveRemoteStream,
        onAddLocalStream);
    if (widget.intent.initialMessage != null) {
      _callingService.processMessage(jsonDecode(widget.intent.initialMessage));
    } else {
      _callingService.startCall();
    }
  }

  void onAddRemoteStream(MediaStream stream) {
    _remoteRenderer.srcObject = stream;
  }

  void onRemoveRemoteStream(MediaStream steam) {
    _remoteRenderer.srcObject = null;
  }

  void onAddLocalStream(MediaStream stream) {
    _localRenderer.srcObject = stream;
  }

  @override
  void initState() {
    super.initState();
    _localRenderer.initialize();
    _remoteRenderer.initialize();
    _client = StompClient(
      config: StompConfig(
        url: 'ws://${DomainService.getDomainBase()}/stomp',
        onConnect: onConnectCallback,
        onWebSocketError: (dynamic error) => print(error.toString()),
        stompConnectHeaders: {'Authorization': "${widget.intent.authToken}"},
        onDisconnect: (message) => print("disconnected ${message.body}"),),
    );
    _client.activate();
  }

  @override
  Widget build(BuildContext context) {
    return PlatformScaffold(
      pageTitle: "",
      child: Flex(
        direction: Axis.vertical,
        children: [
          Flexible(
            flex: 1,
            child: RTCVideoView(_localRenderer),
          ),
          Flexible(
            flex: 1,
            child: RTCVideoView(_remoteRenderer),
          )
        ],
      ),
    );
  }
}

我在 addRemoteStream 回调的小部件中放置了一个打印语句，它被调用了。所以正在发送某种流。我只是不确定为什么视频没有显示。

Answer 1

所以我的问题是我没有向呼叫者添加排队的候选人。

我加了

sendMessage(answerMessage);
if (_remoteCandidates.isNotEmpty) {
  _remoteCandidates
      .forEach((candidate) => _peerConnection.addCandidate(candidate));
  _remoteCandidates.clear();
}

到processAnswer方法，它工作得很好！

Flutter WebRTC 音频但没有视频

Flutter WebRTC audio but no video

webrtc

flutter

flutter-web