长时间的 stackoverflow 阅读器，第一次海报。

我正在尝试创建一个名为 的 iPad 应用程序CloudWriter .该应用程序的概念是绘制您在云中看到的形状。下载应用程序后，启动时 CloudWriter ，用户将看到实时视频背景(来自后置摄像头)，其上有一个 OpenGL 绘图层。用户将能够打开应用程序，将 iPad 指向天空中的云彩，然后在显示屏上绘制他们所看到的内容。

该应用程序的一个主要功能是让用户记录 session 期间显示器上发生的事情的视频屏幕截图。实时视频源和“绘图” View 将成为平面(合并)视频。

关于当前如何工作的一些假设和背景信息。

- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection

- (UIImage *) imageFromSampleBuffer:(CMSampleBufferRef) sampleBuffer

- (void) drawRect:(CGRect)rect {


    NSDate* start = [NSDate date];
    CGContextRef context = [self createBitmapContextOfSize:self.frame.size];

    //not sure why this is necessary...image renders upside-down and mirrored
    CGAffineTransform flipVertical = CGAffineTransformMake(1, 0, 0, -1, 0, self.frame.size.height);
    CGContextConcatCTM(context, flipVertical);

    if( isRecording)
        [[self.layer presentationLayer] renderInContext:context];

    CGImageRef cgImage = CGBitmapContextCreateImage(context);
    UIImage* background = [UIImage imageWithCGImage: cgImage];
    CGImageRelease(cgImage);

    UIImage *bottomImage = background;


    if(((AVCamCaptureManager *)self.captureManager).currentImage != nil && isVideoBGActive )
    {

        UIImage *image = [((AVCamCaptureManager *)self.mainContentScreen.captureManager).currentImage retain];//[UIImage
        CGSize newSize = background.size;
        UIGraphicsBeginImageContext( newSize );
        // Use existing opacity as is
        if( isRecording )
        {
            if( [self.mainContentScreen isVideoBGActive] && _recording)
            {
                [image drawInRect:CGRectMake(0,0,newSize.width,newSize.height)];
            }
            // Apply supplied opacity

            [bottomImage drawInRect:CGRectMake(0,0,newSize.width,newSize.height) blendMode:kCGBlendModeNormal alpha:1.0];

        }
        UIImage *newImage = UIGraphicsGetImageFromCurrentImageContext();

        UIGraphicsEndImageContext();

        self.currentScreen = newImage;



        [image release];
    }

    if (isRecording) {
        float millisElapsed = [[NSDate date] timeIntervalSinceDate:startedAt] * 1000.0;
        [self writeVideoFrameAtTime:CMTimeMake((int)millisElapsed, 1000)];
    }

    float processingSeconds = [[NSDate date] timeIntervalSinceDate:start];
    float delayRemaining = (1.0 / self.frameRate) - processingSeconds;

    CGContextRelease(context);

    //redraw at the specified framerate
    [self performSelector:@selector(setNeedsDisplay) withObject:nil afterDelay:delayRemaining > 0.0 ? delayRemaining : 0.01];  
}

- (CGContextRef) createBitmapContextOfSize:(CGSize) size {
    CGContextRef    context = NULL;
    CGColorSpaceRef colorSpace = nil;
    int             bitmapByteCount;
    int             bitmapBytesPerRow;

    bitmapBytesPerRow   = (size.width * 4);
    bitmapByteCount     = (bitmapBytesPerRow * size.height);
    colorSpace = CGColorSpaceCreateDeviceRGB();
    if (bitmapData != NULL) {
        free(bitmapData);
    }
    bitmapData = malloc( bitmapByteCount );
    if (bitmapData == NULL) {
        fprintf (stderr, "Memory not allocated!");
        CGColorSpaceRelease( colorSpace );
        return NULL;
    }

    context = CGBitmapContextCreate (bitmapData,
                                     size.width ,
                                     size.height,
                                     8,      // bits per component
                                     bitmapBytesPerRow,
                                     colorSpace,
                                     kCGImageAlphaPremultipliedFirst);

    CGContextSetAllowsAntialiasing(context,NO);
    if (context== NULL) {
        free (bitmapData);
        fprintf (stderr, "Context not created!");
        CGColorSpaceRelease( colorSpace );
        return NULL;
    }

    //CGAffineTransform transform = CGAffineTransformIdentity;
    //transform = CGAffineTransformScale(transform, size.width * .25, size.height * .25);
    //CGAffineTransformScale(transform, 1024, 768);

    CGColorSpaceRelease( colorSpace );

    return context;
}

// Delegate routine that is called when a sample buffer was written
- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
       fromConnection:(AVCaptureConnection *)connection
{

    // Create a UIImage from the sample buffer data
    [self imageFromSampleBuffer:sampleBuffer];
}

// Create a UIImage from sample buffer data - modifed not to return a UIImage *, rather store it in self.currentImage
- (UIImage *) imageFromSampleBuffer:(CMSampleBufferRef) sampleBuffer
{

    // unlock the memory, do other stuff, but don't forget:

    // Get a CMSampleBuffer's Core Video image buffer for the media data
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    // Lock the base address of the pixel buffer
    CVPixelBufferLockBaseAddress(imageBuffer, 0);

   // uint8_t *tmp = (uint8_t *)CVPixelBufferGetBaseAddress(imageBuffer);
    int bytes = CVPixelBufferGetBytesPerRow(imageBuffer); // determine number of bytes from height * bytes per row
    //void *baseAddress = malloc(bytes);
    size_t height = CVPixelBufferGetHeight(imageBuffer);     
    uint8_t *baseAddress = malloc( bytes * height );
    memcpy( baseAddress, CVPixelBufferGetBaseAddress(imageBuffer), bytes * height );
    size_t width = CVPixelBufferGetWidth(imageBuffer);

    // Create a device-dependent RGB color space
    CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();

    // Create a bitmap graphics context with the sample buffer data
    CGContextRef context = CGBitmapContextCreate(baseAddress, width, height, 8,
                                                 bytes, colorSpace, kCGBitmapByteOrderDefault | kCGImageAlphaPremultipliedFirst);


    // CGContextScaleCTM(context, 0.25, 0.25); //scale down to size
    // Create a Quartz image from the pixel data in the bitmap graphics context
    CGImageRef quartzImage = CGBitmapContextCreateImage(context);
    // Unlock the pixel buffer
    CVPixelBufferUnlockBaseAddress(imageBuffer,0);

    // Free up the context and color space
    CGContextRelease(context);
    CGColorSpaceRelease(colorSpace);
    free(baseAddress);

    self.currentImage = [UIImage imageWithCGImage:quartzImage scale:0.25 orientation:UIImageOrientationUp];

    // Release the Quartz image
    CGImageRelease(quartzImage);


    return nil; 
}

-(void) writeVideoFrameAtTime:(CMTime)time {
    if (![videoWriterInput isReadyForMoreMediaData]) {
        NSLog(@"Not ready for video data");
    }
    else {
        @synchronized (self) {
            UIImage* newFrame = [self.currentScreen retain];
            CVPixelBufferRef pixelBuffer = NULL;
            CGImageRef cgImage = CGImageCreateCopy([newFrame CGImage]);
            CFDataRef image = CGDataProviderCopyData(CGImageGetDataProvider(cgImage));

            if( image == nil )
            {
                [newFrame release];
                CVPixelBufferRelease( pixelBuffer );
                CGImageRelease(cgImage);
                return;
            }

            int status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, avAdaptor.pixelBufferPool, &pixelBuffer);
            if(status != 0){
                //could not get a buffer from the pool
                NSLog(@"Error creating pixel buffer:  status=%d", status);
            }

            // set image data into pixel buffer
            CVPixelBufferLockBaseAddress( pixelBuffer, 0 );
            uint8_t* destPixels = CVPixelBufferGetBaseAddress(pixelBuffer);
            CFDataGetBytes(image, CFRangeMake(0, CFDataGetLength(image)), destPixels);  //XXX:  will work if the pixel buffer is contiguous and has the same bytesPerRow as the input data

            if(status == 0){
                BOOL success = [avAdaptor appendPixelBuffer:pixelBuffer withPresentationTime:time];
                if (!success)
                    NSLog(@"Warning:  Unable to write buffer to video");
            }

            //clean up
            [newFrame release];
            CVPixelBufferUnlockBaseAddress( pixelBuffer, 0 );
            CVPixelBufferRelease( pixelBuffer );
            CFRelease(image);
            CGImageRelease(cgImage);
        }

    }

}

 if( _recording )
        {
            if( [self.mainContentScreen isVideoBGActive] && _recording)
            {
                [image drawInRect:CGRectMake(0,0,newSize.width,newSize.height)];
            }
            // Apply supplied opacity

            [bottomImage drawInRect:CGRectMake(0,0,newSize.width,newSize.height) blendMode:kCGBlendModeNormal alpha:1.0];

        }

最佳答案

如果您想要不错的录制性能，您将需要避免使用 Core Graphics 重新绘制内容。坚持使用纯 OpenGL ES。

您说您已经在 OpenGL ES 中完成了手指绘画，因此您应该能够将其渲染为纹理。实时视频源也可以指向纹理。从那里，您可以根据手指绘画纹理中的 alpha channel 对两者进行叠加混合。

使用 OpenGL ES 2.0 着色器很容易做到这一点。其实我的GPUImage如果您从绘画代码提供渲染的纹理，开源框架可以处理视频捕获和混合部分(请参阅 FilterShowcase 示例应用程序以获取覆盖在视频上的图像示例)。您必须确保绘画使用的是 OpenGL ES 2.0，而不是 1.1，并且它与 GPUImage OpenGL ES 上下文具有相同的共享组，但我将在 CubeExample 应用程序中展示如何做到这一点。

我还通过使用可用的纹理缓存(在 iOS 5.0+ 上)以高性能方式在 GPUImage 中为您处理视频录制。

通过使用类似我的框架并留在 OpenGL ES 中，您应该能够以稳定的 30 FPS 为 720p 视频(iPad 2)或 1080p 视频(iPad 3)录制这种混合。

关于ios - 带有视频背景和绘画叠加性能问题的全视频屏幕捕获，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/12938545/