iOS 基于实时音视频 SDK 实现屏幕共享功能——3

/uploads/files_user1/article/5fc8bfcb24deb805827.png

iOS 基于实时音视频 SDK 实现屏幕共享功能——1

iOS 基于实时音视频 SDK 实现屏幕共享功能——2

iOS 基于实时音视频 SDK 实现屏幕共享功能——3

iOS 基于实时音视频 SDK 实现屏幕共享功能——4

1.5 VideotoolBox 硬编码

//
//  RongRTCVideoEncoder.m
//  SealRTC
//
//  Created by Sun on 2020/5/13.
//  Copyright © 2020 RongCloud. All rights reserved.
//
#import "RongRTCVideoEncoder.h"
#import "helpers.h"
@interface RongRTCVideoEncoder() {
    VTCompressionSessionRef _compressionSession;
    int _frameTime;
}
/**
 settings
 */
@property (nonatomic, strong) RongRTCVideoEncoderSettings *settings;
/**
 callback queue
 */
@property (nonatomic , strong ) dispatch_queue_t callbackQueue;
- (void)sendSpsAndPPSWithSampleBuffer:(CMSampleBufferRef)sampleBuffer;
- (void)sendNaluData:(CMSampleBufferRef)sampleBuffer;
@end
void compressionOutputCallback(void *encoder,
                               void *params,
                               OSStatus status,
                               VTEncodeInfoFlags infoFlags,
                               CMSampleBufferRef sampleBuffer) {
    RongRTCVideoEncoder *videoEncoder = (__bridge RongRTCVideoEncoder *)encoder;
    if (status != noErr) {
        return;
    }
    if (infoFlags & kVTEncodeInfo_FrameDropped) {
        return;
    }
    BOOL isKeyFrame = NO;
    CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0);
    if (attachments != nullptr && CFArrayGetCount(attachments)) {
        CFDictionaryRef attachment = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, 0)) ;
        isKeyFrame = !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync);
    }
    CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(sampleBuffer);
    CMBlockBufferRef contiguous_buffer = nullptr;
    if (!CMBlockBufferIsRangeContiguous(block_buffer, 0, 0)) {
        status = CMBlockBufferCreateContiguous(nullptr, block_buffer, nullptr, nullptr, 0, 0, 0, &contiguous_buffer);
        if (status != noErr) {
            return;
        }
    } else {
        contiguous_buffer = block_buffer;
        CFRetain(contiguous_buffer);
        block_buffer = nullptr;
    }
    size_t block_buffer_size = CMBlockBufferGetDataLength(contiguous_buffer);
    if (isKeyFrame) {
        [videoEncoder sendSpsAndPPSWithSampleBuffer:sampleBuffer];
    }
    if (contiguous_buffer) {
        CFRelease(contiguous_buffer);
    }
    [videoEncoder sendNaluData:sampleBuffer];
}
@implementation RongRTCVideoEncoder
@synthesize settings = _settings;
@synthesize callbackQueue = _callbackQueue;
- (BOOL)configWithSettings:(RongRTCVideoEncoderSettings *)settings onQueue:(nonnull dispatch_queue_t)queue {
    self.settings = settings;
    if (queue) {
        _callbackQueue = queue;
    } else {
        _callbackQueue = dispatch_get_main_queue();
    }
    if ([self resetCompressionSession:settings]) {
        _frameTime = 0;
        return YES;
    } else {
        return NO;
    }
}
- (BOOL)resetCompressionSession:(RongRTCVideoEncoderSettings *)settings {
    [self destroyCompressionSession];
    OSStatus status = VTCompressionSessionCreate(nullptr, settings.width, settings.height, kCMVideoCodecType_H264, nullptr, nullptr, nullptr, compressionOutputCallback, (__bridge void * _Nullable)(self), &_compressionSession);
    if (status != noErr) {
        return NO;
    }
    [self configureCompressionSession:settings];
    return YES;
}
- (void)configureCompressionSession:(RongRTCVideoEncoderSettings *)settings {
    if (_compressionSession) {
        SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true);
        SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
        SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false);
        SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, 10);
        uint32_t targetBps = settings.startBitrate * 1000;
        SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, targetBps);
        SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, settings.maxFramerate);
        int bitRate = settings.width * settings.height * 3 * 4 * 4;
        SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitRate);
        int bitRateLimit = settings.width * settings.height * 3 * 4;
        SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_DataRateLimits, bitRateLimit);
    }
}
- (void)encode:(CMSampleBufferRef)sampleBuffer {
    CVImageBufferRef imageBuffer = (CVImageBufferRef)CMSampleBufferGetImageBuffer(sampleBuffer);
    CMTime pts = CMTimeMake(self->_frameTime++, 1000);
    VTEncodeInfoFlags flags;
    OSStatus res = VTCompressionSessionEncodeFrame(self->_compressionSession,
                                                   imageBuffer,
                                                   pts,
                                                   kCMTimeInvalid,
                                                   NULL, NULL, &flags);
    if (res != noErr) {
        NSLog(@"encode frame error:%d", (int)res);
        VTCompressionSessionInvalidate(self->_compressionSession);
        CFRelease(self->_compressionSession);
        self->_compressionSession = NULL;
        return;
    }
}
- (void)sendSpsAndPPSWithSampleBuffer:(CMSampleBufferRef)sampleBuffer {
    CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
    const uint8_t *sps ;
    const uint8_t *pps;
    size_t spsSize ,ppsSize , spsCount,ppsCount;
    OSStatus spsStatus = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 0, &sps, &spsSize, &spsCount, NULL);
    OSStatus ppsStatus = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 1, &pps, &ppsSize, &ppsCount, NULL);
    if (spsStatus == noErr && ppsStatus == noErr) {
        const char bytes[] = "\x00\x00\x00\x01";
        size_t length = (sizeof bytes) - 1;
        NSMutableData *spsData = [NSMutableData dataWithCapacity:4+ spsSize];
        NSMutableData *ppsData  = [NSMutableData dataWithCapacity:4 + ppsSize];
        [spsData appendBytes:bytes length:length];
        [spsData appendBytes:sps length:spsSize];
        [ppsData appendBytes:bytes length:length];
        [ppsData appendBytes:pps length:ppsSize];
        if (self && self.callbackQueue) {
            dispatch_async(self.callbackQueue, ^{
                if (self.delegate && [self.delegate respondsToSelector:@selector(spsData:ppsData:)]) {
                    [self.delegate spsData:spsData ppsData:ppsData];
                }
            });
        }
    } else {
        NSLog(@"sps status:%@, pps status:%@", @(spsStatus), @(ppsStatus));
    }
}
- (void)sendNaluData:(CMSampleBufferRef)sampleBuffer {
    size_t totalLength = 0;
    size_t lengthAtOffset=0;
    char *dataPointer;
    CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
    OSStatus status1 = CMBlockBufferGetDataPointer(blockBuffer, 0, &lengthAtOffset, &totalLength, &dataPointer);
    if (status1 != noErr) {
        NSLog(@"video encoder error, status = %d", (int)status1);
        return;
    }
    static const int h264HeaderLength = 4;
    size_t bufferOffset = 0;
    while (bufferOffset < totalLength - h264HeaderLength) {
        uint32_t naluLength = 0;
        memcpy(&naluLength, dataPointer + bufferOffset, h264HeaderLength);
        naluLength = CFSwapInt32BigToHost(naluLength);
        const char bytes[] = "\x00\x00\x00\x01";
        NSMutableData *naluData = [NSMutableData dataWithCapacity:4 + naluLength];
        [naluData appendBytes:bytes length:4];
        [naluData appendBytes:dataPointer + bufferOffset + h264HeaderLength length:naluLength];
        dispatch_async(self.callbackQueue, ^{
            if (self.delegate && [self.delegate respondsToSelector:@selector(naluData:)]) {
                [self.delegate naluData:naluData];
            }
        });
        bufferOffset += naluLength + h264HeaderLength;
    }
}
- (void)destroyCompressionSession {
    if (_compressionSession) {
        VTCompressionSessionInvalidate(_compressionSession);
        CFRelease(_compressionSession);
        _compressionSession = nullptr;
    }
}
- (void)dealloc {
    if (_compressionSession) {
        VTCompressionSessionCompleteFrames(_compressionSession, kCMTimeInvalid);
        VTCompressionSessionInvalidate(_compressionSession);
        CFRelease(_compressionSession);
        _compressionSession = NULL;
    }
}
@end

1.6 VideotoolBox 解码

//
//  RongRTCVideoDecoder.m
//  SealRTC
//
//  Created by Sun on 2020/5/14.
//  Copyright © 2020 RongCloud. All rights reserved.
//
#import "RongRTCVideoDecoder.h"
#import <UIKit/UIKit.h>
#import "helpers.h"
@interface RongRTCVideoDecoder() {
    uint8_t *_sps;
    NSUInteger _spsSize;
    uint8_t *_pps;
    NSUInteger _ppsSize;
    CMVideoFormatDescriptionRef _videoFormatDescription;
    VTDecompressionSessionRef _decompressionSession;
}
/**
 settings
 */
@property (nonatomic, strong) RongRTCVideoEncoderSettings *settings;
/**
 callback queue
 */
@property (nonatomic, strong) dispatch_queue_t callbackQueue;
@end
void DecoderOutputCallback(void * CM_NULLABLE decompressionOutputRefCon,
                           void * CM_NULLABLE sourceFrameRefCon,
                           OSStatus status,
                           VTDecodeInfoFlags infoFlags,
                           CM_NULLABLE CVImageBufferRef imageBuffer,
                           CMTime presentationTimeStamp,
                           CMTime presentationDuration ) {
    if (status != noErr) {
        NSLog(@" decoder callback error :%@", @(status));
        return;
    }
    CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *)sourceFrameRefCon;
    *outputPixelBuffer = CVPixelBufferRetain(imageBuffer);
    RongRTCVideoDecoder *decoder = (__bridge RongRTCVideoDecoder *)(decompressionOutputRefCon);
    dispatch_async(decoder.callbackQueue, ^{
        [decoder.delegate didGetDecodeBuffer:imageBuffer];
        CVPixelBufferRelease(imageBuffer);
    });
}
@implementation RongRTCVideoDecoder
@synthesize settings = _settings;
@synthesize callbackQueue = _callbackQueue;
- (BOOL)configWithSettings:(RongRTCVideoEncoderSettings *)settings onQueue:(dispatch_queue_t)queue {
    self.settings = settings;
    if (queue) {
        _callbackQueue = queue;
    } else {
        _callbackQueue = dispatch_get_main_queue();
    }
    return YES;
}
- (BOOL)createVT {
    if (_decompressionSession) {
        return YES;
    }
    const uint8_t * const parameterSetPointers[2] = {_sps, _pps};
    const size_t parameterSetSizes[2] = {_spsSize, _ppsSize};
    int naluHeaderLen = 4;
    OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault, 2, parameterSetPointers, parameterSetSizes, naluHeaderLen, &_videoFormatDescription );
    if (status != noErr) {
        NSLog(@"CMVideoFormatDescriptionCreateFromH264ParameterSets error:%@", @(status));
        return false;
    }
    NSDictionary *destinationImageBufferAttributes =
                                        @{
                                            (id)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange],
                                            (id)kCVPixelBufferWidthKey: [NSNumber numberWithInteger:self.settings.width],
                                            (id)kCVPixelBufferHeightKey: [NSNumber numberWithInteger:self.settings.height],
                                            (id)kCVPixelBufferOpenGLCompatibilityKey: [NSNumber numberWithBool:true]
                                        };
    VTDecompressionOutputCallbackRecord CallBack;
    CallBack.decompressionOutputCallback = DecoderOutputCallback;
    CallBack.decompressionOutputRefCon = (__bridge void * _Nullable)(self);
    status = VTDecompressionSessionCreate(kCFAllocatorDefault, _videoFormatDescription, NULL, (__bridge CFDictionaryRef _Nullable)(destinationImageBufferAttributes), &CallBack, &_decompressionSession);
    if (status != noErr) {
        NSLog(@"VTDecompressionSessionCreate error:%@", @(status));
        return false;
    }
    status = VTSessionSetProperty(_decompressionSession, kVTDecompressionPropertyKey_RealTime,kCFBooleanTrue);
    return YES;
}
- (CVPixelBufferRef)decode:(uint8_t *)frame withSize:(uint32_t)frameSize {
    CVPixelBufferRef outputPixelBuffer = NULL;
    CMBlockBufferRef blockBuffer = NULL;
    CMBlockBufferFlags flag0 = 0;
    OSStatus status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, frame, frameSize, kCFAllocatorNull, NULL, 0, frameSize, flag0, &blockBuffer);
    if (status != kCMBlockBufferNoErr) {
        NSLog(@"VCMBlockBufferCreateWithMemoryBlock code=%d", (int)status);
        CFRelease(blockBuffer);
        return outputPixelBuffer;
    }
    CMSampleBufferRef sampleBuffer = NULL;
    const size_t sampleSizeArray[] = {frameSize};
    status = CMSampleBufferCreateReady(kCFAllocatorDefault, blockBuffer, _videoFormatDescription, 1, 0, NULL, 1, sampleSizeArray, &sampleBuffer);
    if (status != noErr || !sampleBuffer) {
        NSLog(@"CMSampleBufferCreateReady failed status=%d", (int)status);
        CFRelease(blockBuffer);
        return outputPixelBuffer;
    }
    VTDecodeFrameFlags flag1 = kVTDecodeFrame_1xRealTimePlayback;
    VTDecodeInfoFlags infoFlag = kVTDecodeInfo_Asynchronous;
    status = VTDecompressionSessionDecodeFrame(_decompressionSession, sampleBuffer, flag1, &outputPixelBuffer, &infoFlag);
    if (status == kVTInvalidSessionErr) {
        NSLog(@"decode frame error with session err status =%d", (int)status);
        [self resetVT];
    } else  {
        if (status != noErr) {
            NSLog(@"decode frame error with  status =%d", (int)status);
        }
    }
    CFRelease(sampleBuffer);
    CFRelease(blockBuffer);
    return outputPixelBuffer;
}
- (void)resetVT {
    [self destorySession];
    [self createVT];
}
- (void)decode:(NSData *)data {
    uint8_t *frame = (uint8_t*)[data bytes];
    uint32_t length = data.length;
    uint32_t nalSize = (uint32_t)(length - 4);
    uint32_t *pNalSize = (uint32_t *)frame;
    *pNalSize = CFSwapInt32HostToBig(nalSize);
    int type = (frame[4] & 0x1F);
    CVPixelBufferRef pixelBuffer = NULL;
    switch (type) {
        case 0x05:
            if ([self createVT]) {
                pixelBuffer= [self decode:frame withSize:length];
            }
            break;
        case 0x07:
            self->_spsSize = length - 4;
            self->_sps = (uint8_t *)malloc(self->_spsSize);
            memcpy(self->_sps, &frame[4], self->_spsSize);
            break;
        case 0x08:
            self->_ppsSize = length - 4;
            self->_pps = (uint8_t *)malloc(self->_ppsSize);
            memcpy(self->_pps, &frame[4], self->_ppsSize);
            break;
        default:
            if ([self createVT]) {
                pixelBuffer = [self decode:frame withSize:length];
            }
            break;
    }
}
- (void)dealloc {
    [self destorySession];
}
- (void)destorySession {
    if (_decompressionSession) {
        VTDecompressionSessionInvalidate(_decompressionSession);
        CFRelease(_decompressionSession);
        _decompressionSession = NULL;
    }
}
@end


0 个评论

要回复文章请先登录注册