iOS 基于实时音视频 SDK 实现屏幕共享功能——1
Replaykit 介绍
在之前的 iOS 版本中，iOS 开发者只能拿到编码后的数据，拿不到原始的 PCM 和 YUV，到 iOS 10 之后，开发者可以拿到原始数据，但是只能录制 App 内的内容，如果切到后台，将停止录制，直到 iOS 11，苹果对屏幕共享进行了升级并开放了权限，既可以拿到原始数据，又可以录制整个系统，以下我们重点来说 iOS 11 之后的屏幕共享功能。
系统屏幕共享
- (void)initMode_1 {
self.systemBroadcastPickerView = [[RPSystemBroadcastPickerView alloc] initWithFrame:CGRectMake(0, 64, ScreenWidth, 80)];
self.systemBroadcastPickerView.preferredExtension = @"cn.rongcloud.replaytest.Recoder";
self.systemBroadcastPickerView.backgroundColor = [UIColor colorWithRed:53.0/255.0 green:129.0/255.0 blue:242.0/255.0 alpha:1.0];
self.systemBroadcastPickerView.showsMicrophoneButton = NO;
[self.view addSubview:self.systemBroadcastPickerView];
}在 iOS 11 创建一个 Extension 之后，调用上面的代码就可以开启屏幕共享了，然后系统会为我们生成一个 SampleHandler 的类，在这个方法中，苹果会根据 RPSampleBufferType 上报不同类型的数据。
- (void)processSampleBuffer:(CMSampleBufferRef)sampleBuffer withType:(RPSampleBufferType)sampleBufferType
那怎么通过融云的 RongRTCLib 将屏幕共享数据发送出去呢？
1. 基于 Socket 的逼格玩法
1.1. Replaykit 框架启动和创建 Socket
//
// ViewController.m
// Socket_Replykit
//
// Created by Sun on 2020/5/19.
// Copyright © 2020 RongCloud. All rights reserved.
//
#import "ViewController.h"
#import <ReplayKit/ReplayKit.h>
#import "RongRTCServerSocket.h"
@interface ViewController ()<RongRTCServerSocketProtocol>
@property (nonatomic, strong) RPSystemBroadcastPickerView *systemBroadcastPickerView;
/**
server socket
*/
@property(nonatomic , strong)RongRTCServerSocket *serverSocket;
@end
@implementation ViewController
- (void)viewDidLoad {
[super viewDidLoad];
self.view.backgroundColor = [UIColor whiteColor];
// Do any additional setup after loading the view.
[self.serverSocket createServerSocket];
self.systemBroadcastPickerView = [[RPSystemBroadcastPickerView alloc] initWithFrame:CGRectMake(0, 64, [UIScreen mainScreen].bounds.size.width, 80)];
self.systemBroadcastPickerView.preferredExtension = @"cn.rongcloud.sealrtc.RongRTCRP";
self.systemBroadcastPickerView.backgroundColor = [UIColor colorWithRed:53.0/255.0 green:129.0/255.0 blue:242.0/255.0 alpha:1.0];
self.systemBroadcastPickerView.showsMicrophoneButton = NO;
[self.view addSubview:self.systemBroadcastPickerView];
}
- (RongRTCServerSocket *)serverSocket {
if (!_serverSocket) {
RongRTCServerSocket *socket = [[RongRTCServerSocket alloc] init];
socket.delegate = self;
_serverSocket = socket;
}
return _serverSocket;
}
- (void)didProcessSampleBuffer:(CMSampleBufferRef)sampleBuffer {
// 这里拿到了最终的数据，比如最后可以使用融云的音视频SDK RTCLib 进行传输就可以了
}
@end其中，包括了创建 Server Socket 的步骤，我们把主 App 当做 Server，然后屏幕共享 Extension 当做 Client ，通过 Socket 向我们的主 APP 发送数据。
在 Extension 里面，我们拿到 ReplayKit 框架上报的屏幕视频数据后：
//
// SampleHandler.m
// SocketReply
//
// Created by Sun on 2020/5/19.
// Copyright © 2020 RongCloud. All rights reserved.
//
#import "SampleHandler.h"
#import "RongRTCClientSocket.h"
@interface SampleHandler()
/**
Client Socket
*/
@property (nonatomic, strong) RongRTCClientSocket *clientSocket;
@end
@implementation SampleHandler
- (void)broadcastStartedWithSetupInf(NSDictionary<NSString *,NSObject *> *)setupInfo {
// User has requested to start the broadcast. Setup info from the UI extension can be supplied but optional.
self.clientSocket = [[RongRTCClientSocket alloc] init];
[self.clientSocket createCliectSocket];
}
- (void)processSampleBuffer:(CMSampleBufferRef)sampleBuffer withType:(RPSampleBufferType)sampleBufferType {
switch (sampleBufferType) {
case RPSampleBufferTypeVide
// Handle video sample buffer
[self sendData:sampleBuffer];
break;
case RPSampleBufferTypeAudioApp:
// Handle audio sample buffer for app audio
break;
case RPSampleBufferTypeAudioMic:
// Handle audio sample buffer for mic audio
break;
default:
break;
}
}
- (void)sendData:(CMSampleBufferRef)sampleBuffer {
[self.clientSocket encodeBuffer:sampleBuffer];
}
@end可见 ，这里我们创建了一个 Client Socket，然后拿到屏幕共享的视频 sampleBuffer 之后，通过 Socket 发给我们的主 App，这就是屏幕共享的流程。
1.2 Local Socket 的使用
//
// RongRTCSocket.m
// SealRTC
//
// Created by Sun on 2020/5/7.
// Copyright © 2020 RongCloud. All rights reserved.
//
#import "RongRTCSocket.h"
#import <arpa/inet.h>
#import <netdb.h>
#import <sys/types.h>
#import <sys/socket.h>
#import <ifaddrs.h>
#import "RongRTCThread.h"
@interface RongRTCSocket()
/**
receive thread
*/
@property (nonatomic, strong) RongRTCThread *receiveThread;
@end
@implementation RongRTCSocket
- (int)createSocket {
int socket = socket(AF_INET, SOCK_STREAM, 0);
self.socket = socket;
if (self.socket == -1) {
close(self.socket);
NSLog(@"socket error : %d", self.socket);
}
self.receiveThread = [[RongRTCThread alloc] init];
[self.receiveThread run];
return socket;
}
- (void)setSendBuffer {
int optVal = 1024 * 1024 * 2;
int optLen = sizeof(int);
int res = setsockopt(self.socket, SOL_SOCKET, SO_SNDBUF, (char *)&optVal,optLen);
NSLog(@"set send buffer:%d", res);
}
- (void)setReceiveBuffer {
int optVal = 1024 * 1024 * 2;
int optLen = sizeof(int);
int res = setsockopt(self.socket, SOL_SOCKET, SO_RCVBUF, (char*)&optVal,optLen );
NSLog(@"set send buffer:%d",res);
}
- (void)setSendingTimeout {
struct timeval timeout = {10,0};
int res = setsockopt(self.socket, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(int));
NSLog(@"set send timeout:%d", res);
}
- (void)setReceiveTimeout {
struct timeval timeout = {10, 0};
int res = setsockopt(self.socket, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(int));
NSLog(@"set send timeout:%d", res);
}
- (BOOL)connect {
NSString *serverHost = [self ip];
struct hostent *server = gethostbyname([serverHost UTF8String]);
if (server == NULL) {
close(self.socket);
NSLog(@"get host error");
return NO;
}
struct in_addr *remoteAddr = (struct in_addr *)server->h_addr_list[0];
struct sockaddr_in addr;
addr.sin_family = AF_INET;
addr.sin_addr = *remoteAddr;
addr.sin_port = htons(CONNECTPORT);
int res = connect(self.socket, (struct sockaddr *) &addr, sizeof(addr));
if (res == -1) {
close(self.socket);
NSLog(@"connect error");
return NO;
}
NSLog(@"socket connect to server success");
return YES;
}
- (BOOL)bind {
struct sockaddr_in client;
client.sin_family = AF_INET;
NSString *ipStr = [self ip];
if (ipStr.length <= 0) {
return NO;
}
const char *ip = [ipStr cStringUsingEncoding:NSASCIIStringEncoding];
client.sin_addr.s_addr = inet_addr(ip);
client.sin_port = htons(CONNECTPORT);
int bd = bind(self.socket, (struct sockaddr *) &client, sizeof(client));
if (bd == -1) {
close(self.socket);
NSLog(@"bind error: %d", bd);
return NO;
}
return YES;
}
- (BOOL)listen {
int ls = listen(self.socket, 128);
if (ls == -1) {
close(self.socket);
NSLog(@"listen error: %d", ls);
return NO;
}
return YES;
}
- (void)receive {
dispatch_async(dispatch_get_global_queue(0, 0), ^{
[self receiveData];
});
}
- (NSString *)ip {
NSString *ip = nil;
struct ifaddrs *addrs = NULL;
struct ifaddrs *tmpAddrs = NULL;
BOOL res = getifaddrs(&addrs);
if (res == 0) {
tmpAddrs = addrs;
while (tmpAddrs != NULL) {
if (tmpAddrs->ifa_addr->sa_family == AF_INET) {
// Check if interface is en0 which is the wifi connection on the iPhone
NSLog(@"%@", [NSString stringWithUTF8String:inet_ntoa(((struct sockaddr_in *)tmpAddrs->ifa_addr)->sin_addr)]);
if ([[NSString stringWithUTF8String:tmpAddrs->ifa_name] isEqualToString:@"en0"]) {
// Get NSString from C String
ip = [NSString stringWithUTF8String:inet_ntoa(((struct sockaddr_in *)tmpAddrs->ifa_addr)->sin_addr)];
}
}
tmpAddrs = tmpAddrs->ifa_next;
}
}
// Free memory
freeifaddrs(addrs);
NSLog(@"%@",ip);
return ip;
}
- (void)close {
int res = close(self.socket);
NSLog(@"shut down: %d", res);
}
- (void)receiveData {
}
- (void)dealloc {
[self.receiveThread stop];
}
@end首先创建了一个 Socket 的父类，然后用 Server Socket 和 Client Socket 分别继承类来实现链接、绑定等操作。可以看到有些数据可以设置，有些则不用，这里不是核心，核心是怎样收发数据。
1.3 发送屏幕共享数据
//
// RongRTCClientSocket.m
// SealRTC
//
// Created by Sun on 2020/5/7.
// Copyright © 2020 RongCloud. All rights reserved.
//
#import "RongRTCClientSocket.h"
#import <arpa/inet.h>
#import <netdb.h>
#import <sys/types.h>
#import <sys/socket.h>
#import <ifaddrs.h>
#import "RongRTCThread.h"
#import "RongRTCSocketHeader.h"
#import "RongRTCVideoEncoder.h"
@interface RongRTCClientSocket() <RongRTCCodecProtocol> {
pthread_mutex_t lock;
}
/**
video encoder
*/
@property (nonatomic, strong) RongRTCVideoEncoder *encoder;
/**
encode queue
*/
@property (nonatomic, strong) dispatch_queue_t encodeQueue;
@end
@implementation RongRTCClientSocket
- (BOOL)createClientSocket {
if ([self createSocket] == -1) {
return NO;
}
BOOL isC = [self connect];
[self setSendBuffer];
[self setSendingTimeout];
if (isC) {
_encodeQueue = dispatch_queue_create("cn.rongcloud.encodequeue", NULL);
[self createVideoEncoder];
return YES;
} else {
return NO;
}
}
- (void)createVideoEncoder {
self.encoder = [[RongRTCVideoEncoder alloc] init];
self.encoder.delegate = self;
RongRTCVideoEncoderSettings *settings = [[RongRTCVideoEncoderSettings alloc] init];
settings.width = 720;
settings.height = 1280;
settings.startBitrate = 300;
settings.maxFramerate = 30;
settings.minBitrate = 1000;
[self.encoder configWithSettings:settings onQueue:_encodeQueue];
}
- (void)clientSend:(NSData *)data {
//data length
NSUInteger dataLength = data.length;
// data header struct
DataHeader dataH;
memset((void *)&dataH, 0, sizeof(dataH));
// pre
PreHeader preH;
memset((void *)&preH, 0, sizeof(preH));
preH.pre[0] = '&';
preH.dataLength = dataLength;
dataH.preH = preH;
// buffer
int headerlength = sizeof(dataH);
int totalLength = dataLength + headerlength;
// srcbuffer
Byte *src = (Byte *)[data bytes];
// send buffer
char *buffer = (char *)malloc(totalLength * sizeof(char));
memcpy(buffer, &dataH, headerlength);
memcpy(buffer + headerlength, src, dataLength);
// tosend
[self sendBytes:buffer length:totalLength];
free(buffer);
}
- (void)encodeBuffer:(CMSampleBufferRef)sampleBuffer {
[self.encoder encode:sampleBuffer];
}
- (void)sendBytes:(char *)bytes length:(int)length {
LOCK(self->lock);
int hasSendLength = 0;
while (hasSendLength < length) {
// connect socket success
if (self.socket > 0) {
// send
int sendRes = send(self.socket, bytes, length - hasSendLength, 0);
if (sendRes == -1 || sendRes == 0) {
UNLOCK(self->lock);
NSLog(@"send buffer error");
[self close];
break;
}
hasSendLength += sendRes;
bytes += sendRes;
} else {
NSLog(@"client socket connect error");
UNLOCK(self->lock);
}
}
UNLOCK(self->lock);
}
- (void)spsData:(NSData *)sps ppsData:(NSData *)pps {
[self clientSend:sps];
[self clientSend:pps];
}
- (void)naluData:(NSData *)naluData {
[self clientSend:naluData];
}
- (void)deallo c{
NSLog(@"dealoc cliect socket");
}
@end
