I am constructing a prototype app that showcases the performance on iOS to seize a tool’s display screen, join to a different system, after which video stream this (as excessive res and low latency) over a neighborhood WiFi community. The chosen lib I am implementing this with is WebRTC (open to different ideas).
I load the app onto 2 units, I broadcast the display screen of 1 system, it seems from the logs to hook up with the opposite system I’ve set to obtain, however after that they seem to drop reference to “[GCKSession] Not in linked state, so giving up for participant [xxxxxxxx] on channel [x]” on 5 channel.
I’ve included the complete code you’ll be able to run in Xcode beneath together with a console from the connecting system over the last construct and dropped connection.
I am at a irritating DEAD END.
WebRTCManager.swift
import Basis
import WebRTC
import ReplayKit
import MultipeerConnectivity
class WebRTCManager: NSObject, ObservableObject {
non-public var peerConnection: RTCPeerConnection?
@Printed var localVideoTrack: RTCVideoTrack?
@Printed var remoteVideoTrack: RTCVideoTrack?
non-public var peerConnectionFactory: RTCPeerConnectionFactory?
non-public var videoSource: RTCVideoSource?
non-public var videoCapturer: RTCVideoCapturer?
non-public var peerID: MCPeerID
non-public var session: MCSession
non-public var advertiser: MCNearbyServiceAdvertiser?
non-public var browser: MCNearbyServiceBrowser?
@Printed var connectedPeers: [MCPeerID] = []
@Printed var localSDP: String = ""
@Printed var localICECandidates: [String] = []
@Printed var isBroadcasting: Bool = false
@Printed var remoteTrackAdded: Bool = false
non-public var isConnected = false
override init() {
RTCInitializeSSL()
peerConnectionFactory = RTCPeerConnectionFactory()
peerID = MCPeerID(displayName: UIDevice.present.title)
session = MCSession(peer: peerID, securityIdentity: nil, encryptionPreference: .none)
tremendous.init()
session.delegate = self
}
func startBroadcasting() {
isBroadcasting = true
advertiser = MCNearbyServiceAdvertiser(peer: peerID, discoveryInfo: nil, serviceType: "screen-share")
advertiser?.delegate = self
advertiser?.startAdvertisingPeer()
setupPeerConnection()
setupVideoSource()
startScreenCapture()
}
func startReceiving() {
browser = MCNearbyServiceBrowser(peer: peerID, serviceType: "screen-share")
browser?.delegate = self
browser?.startBrowsingForPeers()
setupPeerConnection()
}
non-public func setupPeerConnection() {
let configuration = RTCConfiguration()
configuration.iceServers = [RTCIceServer(urlStrings: ["stun:stun.l.google.com:19302"])]
configuration.sdpSemantics = .unifiedPlan
let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil)
peerConnection = peerConnectionFactory?.peerConnection(with: configuration, constraints: constraints, delegate: self)
// Guarantee peer connection state is checked earlier than continuing
guard let peerConnection = peerConnection else {
print("Didn't create peer connection")
return
}
peerConnection.delegate = self
}
non-public func setupVideoSource() {
videoSource = peerConnectionFactory?.videoSource()
#if targetEnvironment(simulator)
videoCapturer = RTCFileVideoCapturer(delegate: videoSource!)
#else
videoCapturer = RTCCameraVideoCapturer(delegate: videoSource!)
#endif
localVideoTrack = peerConnectionFactory?.videoTrack(with: videoSource!, trackId: "video0")
if let localVideoTrack = localVideoTrack {
peerConnection?.add(localVideoTrack, streamIds: ["stream0"])
print("Native video observe added to look connection")
} else {
print("Didn't create native video observe")
}
}
non-public func startScreenCapture() {
let recorder = RPScreenRecorder.shared()
recorder.startCapture { [weak self] (sampleBuffer, sort, error) in
guard let self = self else { return }
if let error = error {
print("Error beginning display screen seize: (error)")
return
}
if sort == .video {
self.processSampleBuffer(sampleBuffer, with: sort)
}
} completionHandler: { error in
if let error = error {
print("Error in display screen seize completion: (error)")
} else {
print("Display seize began efficiently")
}
}
}
non-public func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
change sampleBufferType {
case .video:
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
let timestamp = NSDate().timeIntervalSince1970 * 1000
let videoFrame = RTCVideoFrame(buffer: RTCCVPixelBuffer(pixelBuffer: pixelBuffer),
rotation: ._0,
timeStampNs: Int64(timestamp))
self.videoSource?.capturer(self.videoCapturer!, didCapture: videoFrame)
default:
break
}
}
func stopBroadcasting() {
isBroadcasting = false
advertiser?.stopAdvertisingPeer()
advertiser = nil
stopScreenCapture()
closePeerConnection()
}
func stopReceiving() {
browser?.stopBrowsingForPeers()
browser = nil
closePeerConnection()
}
non-public func stopScreenCapture() {
RPScreenRecorder.shared().stopCapture { [weak self] error in
if let error = error {
print("Error stopping display screen seize: (error)")
} else {
print("Display seize stopped efficiently")
DispatchQueue.foremost.async {
self?.localVideoTrack = nil
}
}
}
}
non-public func closePeerConnection() {
guard let peerConnection = peerConnection else { return }
if isConnected {
// Correctly deal with disconnection if linked
peerConnection.shut()
}
self.peerConnection = nil
DispatchQueue.foremost.async {
self.remoteVideoTrack = nil
self.localSDP = ""
self.localICECandidates.removeAll()
}
print("Peer connection closed")
}
non-public func createOffer() {
print("Creating provide")
let constraints = RTCMediaConstraints(mandatoryConstraints: [
"OfferToReceiveVideo": "true",
"OfferToReceiveAudio": "false"
], optionalConstraints: nil)
peerConnection?.provide(for: constraints) { [weak self] sdp, error in
guard let self = self, let sdp = sdp else {
print("Didn't create provide: (error?.localizedDescription ?? "unknown error")")
return
}
self.peerConnection?.setLocalDescription(sdp) { error in
if let error = error {
print("Error setting native description: (error)")
} else {
print("Native description (provide) set efficiently")
self.sendSDP(sdp)
}
}
}
}
non-public func createAnswer() {
print("Creating reply")
let constraints = RTCMediaConstraints(mandatoryConstraints: [
"OfferToReceiveVideo": "true",
"OfferToReceiveAudio": "false"
], optionalConstraints: nil)
peerConnection?.reply(for: constraints) { [weak self] sdp, error in
guard let self = self, let sdp = sdp else {
print("Didn't create reply: (error?.localizedDescription ?? "unknown error")")
return
}
self.peerConnection?.setLocalDescription(sdp) { error in
if let error = error {
print("Error setting native description: (error)")
} else {
print("Native description (reply) set efficiently")
self.sendSDP(sdp)
}
}
}
}
non-public func setRemoteDescription(_ sdp: RTCSessionDescription) {
peerConnection?.setRemoteDescription(sdp) { error in
if let error = error {
print("Error setting distant description: (error)")
} else {
print("Distant description set efficiently")
}
}
}
non-public func addIceCandidate(_ candidate: RTCIceCandidate) {
guard let peerConnection = peerConnection, isConnected else {
print("Can not add ICE candidate, peer connection will not be linked")
return
}
peerConnection.add(candidate)
print("ICE candidate added")
}
non-public func sendSDP(_ sdp: RTCSessionDescription) {
let dict: [String: Any] = ["type": sdp.type.rawValue, "sdp": sdp.sdp]
if let information = strive? JSONSerialization.information(withJSONObject: dict, choices: []) {
do {
strive session.ship(information, toPeers: session.connectedPeers, with: .dependable)
print("SDP despatched to friends")
} catch {
print("Didn't ship SDP: (error)")
}
}
}
}
extension WebRTCManager: MCSessionDelegate, MCNearbyServiceAdvertiserDelegate, MCNearbyServiceBrowserDelegate {
func session(_ session: MCSession, peer peerID: MCPeerID, didChange state: MCSessionState) {
DispatchQueue.foremost.async {
self.connectedPeers = session.connectedPeers
change state {
case .linked:
print("Peer linked: (peerID.displayName)")
self.browser?.stopBrowsingForPeers()
self.advertiser?.stopAdvertisingPeer()
self.isConnected = true // Set isConnected to true right here
if self.isBroadcasting {
self.createOffer()
}
case .connecting:
print("Peer connecting: (peerID.displayName)")
case .notConnected:
print("Peer not linked: (peerID.displayName)")
self.isConnected = false
@unknown default:
print("Unknown state: (peerID.displayName)")
}
}
}
func session(_ session: MCSession, didReceive information: Information, fromPeer peerID: MCPeerID) {
let dict = strive? JSONSerialization.jsonObject(with: information, choices: []) as? [String: Any]
if let typeInt = dict?["type"] as? Int, let sdp = dict?["sdp"] as? String,
let sort = RTCSdpType(rawValue: typeInt) {
let rtcSdp = RTCSessionDescription(sort: sort, sdp: sdp)
self.peerConnection?.setRemoteDescription(rtcSdp) { [weak self] error in
if let error = error {
print("Error setting distant description: (error)")
} else {
print("Distant description set efficiently")
if sort == .provide {
self?.createAnswer()
}
}
}
} else if let sdp = dict?["candidate"] as? String,
let sdpMid = dict?["sdpMid"] as? String,
let sdpMLineIndexString = dict?["sdpMLineIndex"] as? String,
let sdpMLineIndex = Int32(sdpMLineIndexString) {
let candidate = RTCIceCandidate(sdp: sdp, sdpMLineIndex: sdpMLineIndex, sdpMid: sdpMid)
self.peerConnection?.add(candidate)
}
}
func session(_ session: MCSession, didReceive stream: InputStream, withName streamName: String, fromPeer peerID: MCPeerID) {}
func session(_ session: MCSession, didStartReceivingResourceWithName resourceName: String, fromPeer peerID: MCPeerID, with progress: Progress) {}
func session(_ session: MCSession, didFinishReceivingResourceWithName resourceName: String, fromPeer peerID: MCPeerID, at localURL: URL?, withError error: Error?) {}
func advertiser(_ advertiser: MCNearbyServiceAdvertiser, didReceiveInvitationFromPeer peerID: MCPeerID, withContext context: Information?, invitationHandler: @escaping (Bool, MCSession?) -> Void) {
invitationHandler(true, session)
}
func browser(_ browser: MCNearbyServiceBrowser, foundPeer peerID: MCPeerID, withDiscoveryInfo data: [String : String]?) {
browser.invitePeer(peerID, to: session, withContext: nil, timeout: 30)
}
func browser(_ browser: MCNearbyServiceBrowser, lostPeer peerID: MCPeerID) {}
}
extension WebRTCManager: RTCPeerConnectionDelegate {
func peerConnection(_ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState) {
print("Signaling state modified: (stateChanged.rawValue)")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {
print("Stream added with ID: (stream.streamId)")
if let videoTrack = stream.videoTracks.first {
print("Video observe added: (videoTrack.trackId)")
DispatchQueue.foremost.async {
self.remoteVideoTrack = videoTrack
self.remoteTrackAdded = true
self.objectWillChange.ship()
print("Distant video observe set")
}
} else {
print("No video tracks within the stream")
}
}
func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {
print("Stream eliminated")
}
func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) {
print("Negotiation wanted")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
print("ICE connection state modified: (newState.rawValue)")
change newState {
case .checking, .linked, .accomplished:
print("ICE linked")
self.isConnected = true
case .failed, .disconnected, .closed:
print("ICE connection failed or closed")
self.isConnected = false
// Deal with reconnection or cleanup if mandatory
case .new:
print("New ICE connection")
case .rely:
print("ICE rely")
@unknown default:
print("Unknown ICE state")
}
}
func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState) {
print("ICE gathering state modified: (newState.rawValue)")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) {
print("ICE candidate generated: (candidate.sdp)")
DispatchQueue.foremost.async {
self.localICECandidates.append(candidate.sdp)
}
// All the time ship ICE candidates
let dict: [String: Any] = ["candidate": candidate.sdp, "sdpMid": candidate.sdpMid ?? "", "sdpMLineIndex": candidate.sdpMLineIndex]
if let information = strive? JSONSerialization.information(withJSONObject: dict, choices: []) {
do {
strive session.ship(information, toPeers: session.connectedPeers, with: .dependable)
print("ICE candidate despatched to friends")
} catch {
print("Didn't ship ICE candidate: (error)")
}
}
}
func peerConnection(_ peerConnection: RTCPeerConnection, didRemove candidates: [RTCIceCandidate]) {
print("Eliminated ICE candidates")
}
func peerConnection(_ peerConnection: RTCPeerConnection, didOpen dataChannel: RTCDataChannel) {
print("Information channel opened")
}
}
extension Notification.Identify {
static let remoteVideoTrackAdded = Notification.Identify("remoteVideoTrackAdded")
}
RTCVideoView.swift
import SwiftUI
import WebRTC
struct RTCVideoView: UIViewRepresentable {
@ObservedObject var webRTCManager: WebRTCManager
var isLocal: Bool
func makeUIView(context: Context) -> RTCMTLVideoView {
let videoView = RTCMTLVideoView(body: .zero)
videoView.videoContentMode = .scaleAspectFit
updateVideoTrack(videoView)
return videoView
}
func updateUIView(_ uiView: RTCMTLVideoView, context: Context) {
updateVideoTrack(uiView)
}
non-public func updateVideoTrack(_ uiView: RTCMTLVideoView) {
if isLocal {
if let localVideoTrack = webRTCManager.localVideoTrack {
localVideoTrack.add(uiView)
print("Native video observe added to view")
} else {
print("Native video observe is nil")
}
} else {
if let remoteVideoTrack = webRTCManager.remoteVideoTrack {
remoteVideoTrack.add(uiView)
print("Distant video observe added to view")
} else {
print("Distant video observe is nil")
}
}
}
func makeCoordinator() -> Coordinator {
Coordinator(self)
}
class Coordinator: NSObject {
var dad or mum: RTCVideoView
init(_ dad or mum: RTCVideoView) {
self.dad or mum = dad or mum
tremendous.init()
NotificationCenter.default.addObserver(self, selector: #selector(remoteVideoTrackAdded), title: .remoteVideoTrackAdded, object: nil)
}
@objc func remoteVideoTrackAdded() {
DispatchQueue.foremost.async {
self.dad or mum.webRTCManager.objectWillChange.ship()
}
}
}
}
ContentView.swift
import SwiftUI
import WebRTC
import MultipeerConnectivity
import ReplayKit
struct ContentView: View {
@StateObject non-public var webRTCManager = WebRTCManager()
@State non-public var isBroadcasting = false
@State non-public var isReceiving = false
var physique: some View {
VStack {
if isBroadcasting {
Textual content("Broadcasting")
.font(.headline)
// Native video preview
RTCVideoView(webRTCManager: webRTCManager, isLocal: true)
.body(peak: 200)
.background(Coloration.grey.opacity(0.3)) // Add a semi-transparent grey background
.cornerRadius(10)
Button("Cease Broadcasting") {
webRTCManager.stopBroadcasting()
isBroadcasting = false
}
.padding()
.background(Coloration.pink)
.foregroundColor(.white)
.cornerRadius(10)
// ... (present code for SDP and ICE candidates)
} else if isReceiving {
Textual content("Receiving")
.font(.headline)
RTCVideoView(webRTCManager: webRTCManager, isLocal: false)
.body(peak: 300)
.background(Coloration.grey.opacity(0.3)) // Add a semi-transparent grey background
.cornerRadius(10)
Button("Cease Receiving") {
webRTCManager.stopReceiving()
isReceiving = false
}
.padding()
.background(Coloration.pink)
.foregroundColor(.white)
.cornerRadius(10)
} else {
Button("Begin Broadcasting") {
webRTCManager.startBroadcasting()
isBroadcasting = true
}
.padding()
.background(Coloration.blue)
.foregroundColor(.white)
.cornerRadius(10)
Button("Begin Receiving") {
webRTCManager.startReceiving()
isReceiving = true
}
.padding()
.background(Coloration.inexperienced)
.foregroundColor(.white)
.cornerRadius(10)
}
Textual content("Linked Friends: (webRTCManager.connectedPeers.rely)")
.font(.headline)
.padding()
if !webRTCManager.connectedPeers.isEmpty {
Textual content("Linked to:")
ForEach(webRTCManager.connectedPeers, id: .self) { peer in
Textual content(peer.displayName)
}
}
}
.padding()
}
}
struct ContentView_Previews: PreviewProvider {
static var previews: some View {
ContentView()
}
}
Console of connecting system:
2024-07-08 22:04:22.574897+0100 ScreenShare[2745:914162] Steel API Validation Enabled
Distant video observe is nil
Distant video observe is nil
2024-07-08 22:04:56.058516+0100 ScreenShare[2745:914299] [Client] Updating selectors after delegate removing failed with: Error Area=NSCocoaErrorDomain Code=4099 "The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this course of." UserInfo={NSDebugDescription=The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this course of.}
2024-07-08 22:04:56.058657+0100 ScreenShare[2745:914299] [Client] Updating selectors after delegate addition failed with: Error Area=NSCocoaErrorDomain Code=4099 "The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this course of." UserInfo={NSDebugDescription=The connection to service named com.apple.commcenter.coretelephony.xpc was invalidated from this course of.}
Peer connecting: iPhone
Distant video observe is nil
Peer linked: iPhone
Distant video observe is nil
Signaling state modified: 3
Stream added with ID: stream0
Video observe added: video0
Distant description set efficiently
Creating reply
Distant video observe set
Distant video observe added to view
Signaling state modified: 0
Native description (reply) set efficiently
SDP despatched to friends
ICE gathering state modified: 1
ICE candidate generated: candidate:617392483 1 udp 2122260223 192.168.1.112 63716 typ host era 0 ufrag umCW network-id 1 network-cost 10
ICE candidate despatched to friends
ICE candidate generated: candidate:3503244297 1 udp 2122194687 169.254.104.95 64683 typ host era 0 ufrag umCW network-id 2 network-cost 10
ICE candidate despatched to friends
Distant video observe added to view
Distant video observe added to view
ICE candidate generated: candidate:1783584147 1 tcp 1518280447 192.168.1.112 51096 typ host tcptype passive era 0 ufrag umCW network-id 1 network-cost 10
ICE candidate despatched to friends
ICE candidate generated: candidate:2655828217 1 tcp 1518214911 169.254.104.95 51097 typ host tcptype passive era 0 ufrag umCW network-id 2 network-cost 10
ICE candidate despatched to friends
Distant video observe added to view
ICE candidate generated: candidate:2776936407 1 udp 1686052607 2.28.217.67 63716 typ srflx raddr 192.168.1.112 rport 63716 era 0 ufrag umCW network-id 1 network-cost 10
ICE candidate despatched to friends
Distant video observe added to view
Distant video observe added to view
2024-07-08 22:05:06.151870+0100 ScreenShare[2745:914169] [GCKSession] Not in linked state, so giving up for participant [780785AF] on channel [0].
2024-07-08 22:05:06.155864+0100 ScreenShare[2745:914169] [GCKSession] Not in linked state, so giving up for participant [780785AF] on channel [1].
2024-07-08 22:05:06.158066+0100 ScreenShare[2745:914169] [GCKSession] Not in linked state, so giving up for participant [780785AF] on channel [2].
2024-07-08 22:05:06.159428+0100 ScreenShare[2745:914169] [GCKSession] Not in linked state, so giving up for participant [780785AF] on channel [3].
2024-07-08 22:05:06.160762+0100 ScreenShare[2745:914169] [GCKSession] Not in linked state, so giving up for participant [780785AF] on channel [4].
2024-07-08 22:05:06.161831+0100 ScreenShare[2745:914169] [GCKSession] Not in linked state, so giving up for participant [780785AF] on channel [5].
2024-07-08 22:05:06.162682+0100 ScreenShare[2745:914169] [GCKSession] Not in linked state, so giving up for participant [780785AF] on channel [6].
ICE gathering state modified: 2
I’ve fiddled with the initiatives settings and added something I’ve discovered might be inflicting the problem — as a result of its absence from the Information.plist — and added it in:
<plist model="1.0">
<dict>
<key>NSBonjourServices</key>
<array>
<string>_screen-share._tcp</string>
<string>_screen-share._udp</string>
</array>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
<false/>
<key>UISceneConfigurations</key>
<dict/>
</dict>
<key>UIBackgroundModes</key>
<array>
<string>audio</string>
<string>fetch</string>
<string>nearby-interaction</string>
<string>processing</string>
<string>voip</string>
</array>
</dict>
</plist>
