iOS 基于实时音视频 SDK 实现屏幕共享功能——1
Replaykit 介绍
在之前的 iOS 版本中,iOS 开发者只能拿到编码后的数据,拿不到原始的 PCM 和 YUV,到 iOS 10 之后,开发者可以拿到原始数据,但是只能录制 App 内的内容,如果切到后台,将停止录制,直到 iOS 11,苹果对屏幕共享进行了升级并开放了权限,既可以拿到原始数据,又可以录制整个系统,以下我们重点来说 iOS 11 之后的屏幕共享功能。
系统屏幕共享
- (void)initMode_1 { self.systemBroadcastPickerView = [[RPSystemBroadcastPickerView alloc] initWithFrame:CGRectMake(0, 64, ScreenWidth, 80)]; self.systemBroadcastPickerView.preferredExtension = @"cn.rongcloud.replaytest.Recoder"; self.systemBroadcastPickerView.backgroundColor = [UIColor colorWithRed:53.0/255.0 green:129.0/255.0 blue:242.0/255.0 alpha:1.0]; self.systemBroadcastPickerView.showsMicrophoneButton = NO; [self.view addSubview:self.systemBroadcastPickerView]; }
在 iOS 11 创建一个 Extension
之后,调用上面的代码就可以开启屏幕共享了,然后系统会为我们生成一个 SampleHandler
的类,在这个方法中,苹果会根据 RPSampleBufferType
上报不同类型的数据。
- (void)processSampleBuffer:(CMSampleBufferRef)sampleBuffer withType:(RPSampleBufferType)sampleBufferType
那怎么通过融云的 RongRTCLib
将屏幕共享数据发送出去呢?
1. 基于 Socket 的逼格玩法
1.1. Replaykit 框架启动和创建 Socket
// // ViewController.m // Socket_Replykit // // Created by Sun on 2020/5/19. // Copyright © 2020 RongCloud. All rights reserved. // #import "ViewController.h" #import <ReplayKit/ReplayKit.h> #import "RongRTCServerSocket.h" @interface ViewController ()<RongRTCServerSocketProtocol> @property (nonatomic, strong) RPSystemBroadcastPickerView *systemBroadcastPickerView; /** server socket */ @property(nonatomic , strong)RongRTCServerSocket *serverSocket; @end @implementation ViewController - (void)viewDidLoad { [super viewDidLoad]; self.view.backgroundColor = [UIColor whiteColor]; // Do any additional setup after loading the view. [self.serverSocket createServerSocket]; self.systemBroadcastPickerView = [[RPSystemBroadcastPickerView alloc] initWithFrame:CGRectMake(0, 64, [UIScreen mainScreen].bounds.size.width, 80)]; self.systemBroadcastPickerView.preferredExtension = @"cn.rongcloud.sealrtc.RongRTCRP"; self.systemBroadcastPickerView.backgroundColor = [UIColor colorWithRed:53.0/255.0 green:129.0/255.0 blue:242.0/255.0 alpha:1.0]; self.systemBroadcastPickerView.showsMicrophoneButton = NO; [self.view addSubview:self.systemBroadcastPickerView]; } - (RongRTCServerSocket *)serverSocket { if (!_serverSocket) { RongRTCServerSocket *socket = [[RongRTCServerSocket alloc] init]; socket.delegate = self; _serverSocket = socket; } return _serverSocket; } - (void)didProcessSampleBuffer:(CMSampleBufferRef)sampleBuffer { // 这里拿到了最终的数据,比如最后可以使用融云的音视频SDK RTCLib 进行传输就可以了 } @end
其中,包括了创建 Server Socket
的步骤,我们把主 App 当做 Server
,然后屏幕共享 Extension
当做 Client
,通过 Socket
向我们的主 APP 发送数据。
在 Extension
里面,我们拿到 ReplayKit
框架上报的屏幕视频数据后:
// // SampleHandler.m // SocketReply // // Created by Sun on 2020/5/19. // Copyright © 2020 RongCloud. All rights reserved. // #import "SampleHandler.h" #import "RongRTCClientSocket.h" @interface SampleHandler() /** Client Socket */ @property (nonatomic, strong) RongRTCClientSocket *clientSocket; @end @implementation SampleHandler - (void)broadcastStartedWithSetupInf(NSDictionary<NSString *,NSObject *> *)setupInfo { // User has requested to start the broadcast. Setup info from the UI extension can be supplied but optional. self.clientSocket = [[RongRTCClientSocket alloc] init]; [self.clientSocket createCliectSocket]; } - (void)processSampleBuffer:(CMSampleBufferRef)sampleBuffer withType:(RPSampleBufferType)sampleBufferType { switch (sampleBufferType) { case RPSampleBufferTypeVide // Handle video sample buffer [self sendData:sampleBuffer]; break; case RPSampleBufferTypeAudioApp: // Handle audio sample buffer for app audio break; case RPSampleBufferTypeAudioMic: // Handle audio sample buffer for mic audio break; default: break; } } - (void)sendData:(CMSampleBufferRef)sampleBuffer { [self.clientSocket encodeBuffer:sampleBuffer]; } @end
可见 ,这里我们创建了一个 Client Socket
,然后拿到屏幕共享的视频 sampleBuffer
之后,通过 Socket
发给我们的主 App,这就是屏幕共享的流程。
1.2 Local Socket 的使用
// // RongRTCSocket.m // SealRTC // // Created by Sun on 2020/5/7. // Copyright © 2020 RongCloud. All rights reserved. // #import "RongRTCSocket.h" #import <arpa/inet.h> #import <netdb.h> #import <sys/types.h> #import <sys/socket.h> #import <ifaddrs.h> #import "RongRTCThread.h" @interface RongRTCSocket() /** receive thread */ @property (nonatomic, strong) RongRTCThread *receiveThread; @end @implementation RongRTCSocket - (int)createSocket { int socket = socket(AF_INET, SOCK_STREAM, 0); self.socket = socket; if (self.socket == -1) { close(self.socket); NSLog(@"socket error : %d", self.socket); } self.receiveThread = [[RongRTCThread alloc] init]; [self.receiveThread run]; return socket; } - (void)setSendBuffer { int optVal = 1024 * 1024 * 2; int optLen = sizeof(int); int res = setsockopt(self.socket, SOL_SOCKET, SO_SNDBUF, (char *)&optVal,optLen); NSLog(@"set send buffer:%d", res); } - (void)setReceiveBuffer { int optVal = 1024 * 1024 * 2; int optLen = sizeof(int); int res = setsockopt(self.socket, SOL_SOCKET, SO_RCVBUF, (char*)&optVal,optLen ); NSLog(@"set send buffer:%d",res); } - (void)setSendingTimeout { struct timeval timeout = {10,0}; int res = setsockopt(self.socket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(int)); NSLog(@"set send timeout:%d", res); } - (void)setReceiveTimeout { struct timeval timeout = {10, 0}; int res = setsockopt(self.socket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(int)); NSLog(@"set send timeout:%d", res); } - (BOOL)connect { NSString *serverHost = [self ip]; struct hostent *server = gethostbyname([serverHost UTF8String]); if (server == NULL) { close(self.socket); NSLog(@"get host error"); return NO; } struct in_addr *remoteAddr = (struct in_addr *)server->h_addr_list[0]; struct sockaddr_in addr; addr.sin_family = AF_INET; addr.sin_addr = *remoteAddr; addr.sin_port = htons(CONNECTPORT); int res = connect(self.socket, (struct sockaddr *) &addr, sizeof(addr)); if (res == -1) { close(self.socket); NSLog(@"connect error"); return NO; } NSLog(@"socket connect to server success"); return YES; } - (BOOL)bind { struct sockaddr_in client; client.sin_family = AF_INET; NSString *ipStr = [self ip]; if (ipStr.length <= 0) { return NO; } const char *ip = [ipStr cStringUsingEncoding:NSASCIIStringEncoding]; client.sin_addr.s_addr = inet_addr(ip); client.sin_port = htons(CONNECTPORT); int bd = bind(self.socket, (struct sockaddr *) &client, sizeof(client)); if (bd == -1) { close(self.socket); NSLog(@"bind error: %d", bd); return NO; } return YES; } - (BOOL)listen { int ls = listen(self.socket, 128); if (ls == -1) { close(self.socket); NSLog(@"listen error: %d", ls); return NO; } return YES; } - (void)receive { dispatch_async(dispatch_get_global_queue(0, 0), ^{ [self receiveData]; }); } - (NSString *)ip { NSString *ip = nil; struct ifaddrs *addrs = NULL; struct ifaddrs *tmpAddrs = NULL; BOOL res = getifaddrs(&addrs); if (res == 0) { tmpAddrs = addrs; while (tmpAddrs != NULL) { if (tmpAddrs->ifa_addr->sa_family == AF_INET) { // Check if interface is en0 which is the wifi connection on the iPhone NSLog(@"%@", [NSString stringWithUTF8String:inet_ntoa(((struct sockaddr_in *)tmpAddrs->ifa_addr)->sin_addr)]); if ([[NSString stringWithUTF8String:tmpAddrs->ifa_name] isEqualToString:@"en0"]) { // Get NSString from C String ip = [NSString stringWithUTF8String:inet_ntoa(((struct sockaddr_in *)tmpAddrs->ifa_addr)->sin_addr)]; } } tmpAddrs = tmpAddrs->ifa_next; } } // Free memory freeifaddrs(addrs); NSLog(@"%@",ip); return ip; } - (void)close { int res = close(self.socket); NSLog(@"shut down: %d", res); } - (void)receiveData { } - (void)dealloc { [self.receiveThread stop]; } @end
首先创建了一个 Socket
的父类,然后用 Server Socket
和 Client Socket
分别继承类来实现链接、绑定等操作。可以看到有些数据可以设置,有些则不用,这里不是核心,核心是怎样收发数据。
1.3 发送屏幕共享数据
// // RongRTCClientSocket.m // SealRTC // // Created by Sun on 2020/5/7. // Copyright © 2020 RongCloud. All rights reserved. // #import "RongRTCClientSocket.h" #import <arpa/inet.h> #import <netdb.h> #import <sys/types.h> #import <sys/socket.h> #import <ifaddrs.h> #import "RongRTCThread.h" #import "RongRTCSocketHeader.h" #import "RongRTCVideoEncoder.h" @interface RongRTCClientSocket() <RongRTCCodecProtocol> { pthread_mutex_t lock; } /** video encoder */ @property (nonatomic, strong) RongRTCVideoEncoder *encoder; /** encode queue */ @property (nonatomic, strong) dispatch_queue_t encodeQueue; @end @implementation RongRTCClientSocket - (BOOL)createClientSocket { if ([self createSocket] == -1) { return NO; } BOOL isC = [self connect]; [self setSendBuffer]; [self setSendingTimeout]; if (isC) { _encodeQueue = dispatch_queue_create("cn.rongcloud.encodequeue", NULL); [self createVideoEncoder]; return YES; } else { return NO; } } - (void)createVideoEncoder { self.encoder = [[RongRTCVideoEncoder alloc] init]; self.encoder.delegate = self; RongRTCVideoEncoderSettings *settings = [[RongRTCVideoEncoderSettings alloc] init]; settings.width = 720; settings.height = 1280; settings.startBitrate = 300; settings.maxFramerate = 30; settings.minBitrate = 1000; [self.encoder configWithSettings:settings onQueue:_encodeQueue]; } - (void)clientSend:(NSData *)data { //data length NSUInteger dataLength = data.length; // data header struct DataHeader dataH; memset((void *)&dataH, 0, sizeof(dataH)); // pre PreHeader preH; memset((void *)&preH, 0, sizeof(preH)); preH.pre[0] = '&'; preH.dataLength = dataLength; dataH.preH = preH; // buffer int headerlength = sizeof(dataH); int totalLength = dataLength + headerlength; // srcbuffer Byte *src = (Byte *)[data bytes]; // send buffer char *buffer = (char *)malloc(totalLength * sizeof(char)); memcpy(buffer, &dataH, headerlength); memcpy(buffer + headerlength, src, dataLength); // tosend [self sendBytes:buffer length:totalLength]; free(buffer); } - (void)encodeBuffer:(CMSampleBufferRef)sampleBuffer { [self.encoder encode:sampleBuffer]; } - (void)sendBytes:(char *)bytes length:(int)length { LOCK(self->lock); int hasSendLength = 0; while (hasSendLength < length) { // connect socket success if (self.socket > 0) { // send int sendRes = send(self.socket, bytes, length - hasSendLength, 0); if (sendRes == -1 || sendRes == 0) { UNLOCK(self->lock); NSLog(@"send buffer error"); [self close]; break; } hasSendLength += sendRes; bytes += sendRes; } else { NSLog(@"client socket connect error"); UNLOCK(self->lock); } } UNLOCK(self->lock); } - (void)spsData:(NSData *)sps ppsData:(NSData *)pps { [self clientSend:sps]; [self clientSend:pps]; } - (void)naluData:(NSData *)naluData { [self clientSend:naluData]; } - (void)deallo c{ NSLog(@"dealoc cliect socket"); } @end