Commit 00dcbffa authored by Kateryna Kostiuk's avatar Kateryna Kostiuk

video: change rendering

- use metal framework for video view and preview rendering
- use avframe to get video data

Change-Id: I8faa6f2e8e529e875af207876aa7445f9712d630
parent 76744e40
...@@ -37,6 +37,8 @@ FIND_PACKAGE(OpenGL REQUIRED) ...@@ -37,6 +37,8 @@ FIND_PACKAGE(OpenGL REQUIRED)
EXECUTE_PROCESS(COMMAND git submodule update --init EXECUTE_PROCESS(COMMAND git submodule update --init
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
get_filename_component(PARENT_DIR ${CMAKE_SOURCE_DIR} PATH)
INCLUDE(ExternalProject) INCLUDE(ExternalProject)
ExternalProject_Add(libqrencode ExternalProject_Add(libqrencode
GIT_SUBMODULES libqrencode GIT_SUBMODULES libqrencode
...@@ -52,6 +54,17 @@ ExternalProject_Add(libqrencode ...@@ -52,6 +54,17 @@ ExternalProject_Add(libqrencode
INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/libqrencode/include) INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/libqrencode/include)
LINK_DIRECTORIES(${CMAKE_SOURCE_DIR}/libqrencode/lib) LINK_DIRECTORIES(${CMAKE_SOURCE_DIR}/libqrencode/lib)
INCLUDE_DIRECTORIES(${PARENT_DIR}/daemon/contrib/native/ffmpeg)
set(SHADERS_FILE "Shader.metallib")
add_custom_command (OUTPUT ${CMAKE_SOURCE_DIR}/Shader.metallib
COMMAND ${CMAKE_SOURCE_DIR}/generateShaderLib.sh
COMMENT "Creating Shader.metallib")
add_custom_target(
shader ALL
DEPENDS ${CMAKE_SOURCE_DIR}/Shader.metallib
)
IF(NOT (${ENABLE_SPARKLE} MATCHES false)) IF(NOT (${ENABLE_SPARKLE} MATCHES false))
MESSAGE("Sparkle auto-update enabled") MESSAGE("Sparkle auto-update enabled")
...@@ -212,6 +225,8 @@ SET(ringclient_VIEWS ...@@ -212,6 +225,8 @@ SET(ringclient_VIEWS
src/views/HoverButton.mm src/views/HoverButton.mm
src/views/CenteredClipView.h src/views/CenteredClipView.h
src/views/CenteredClipView.mm src/views/CenteredClipView.mm
src/views/CallMTKView.h
src/views/CallMTKView.mm
) )
SET(ringclient_OTHERS SET(ringclient_OTHERS
...@@ -229,8 +244,9 @@ SET(ringclient_OTHERS ...@@ -229,8 +244,9 @@ SET(ringclient_OTHERS
src/NSString+Extensions.h src/NSString+Extensions.h
src/NSString+Extensions.mm src/NSString+Extensions.mm
src/RingMainWindow.h src/RingMainWindow.h
src/RingMainWindow.mm) src/RingMainWindow.mm
src/Shader.metal
)
SET(ringclient_XIBS SET(ringclient_XIBS
MainMenu MainMenu
...@@ -323,6 +339,8 @@ SET_SOURCE_FILES_PROPERTIES(${ring_ICONS} PROPERTIES ...@@ -323,6 +339,8 @@ SET_SOURCE_FILES_PROPERTIES(${ring_ICONS} PROPERTIES
MACOSX_PACKAGE_LOCATION Resources) MACOSX_PACKAGE_LOCATION Resources)
SET_SOURCE_FILES_PROPERTIES(Credits.rtf PROPERTIES SET_SOURCE_FILES_PROPERTIES(Credits.rtf PROPERTIES
MACOSX_PACKAGE_LOCATION Resources) MACOSX_PACKAGE_LOCATION Resources)
SET_SOURCE_FILES_PROPERTIES(Shader.metallib PROPERTIES
MACOSX_PACKAGE_LOCATION Resources)
# package ringtones # package ringtones
IF(NOT IS_DIRECTORY ${RINGTONE_DIR}) IF(NOT IS_DIRECTORY ${RINGTONE_DIR})
...@@ -405,6 +423,7 @@ SET(TO_ADD ...@@ -405,6 +423,7 @@ SET(TO_ADD
${LOCALIZABLE_FILES} ${LOCALIZABLE_FILES}
${myApp_ICON} ${myApp_ICON}
Credits.rtf Credits.rtf
Shader.metallib
${ring_ICONS} ${ring_ICONS}
${ring_RINGTONES}) ${ring_RINGTONES})
...@@ -438,6 +457,8 @@ TARGET_LINK_LIBRARIES( ${PROJ_NAME} ...@@ -438,6 +457,8 @@ TARGET_LINK_LIBRARIES( ${PROJ_NAME}
-lqrencode -lqrencode
) )
target_link_libraries(${PROJ_NAME} ${PARENT_DIR}/daemon/contrib/x86_64-apple-darwin${CMAKE_SYSTEM_VERSION}/lib/libavutil.a)
IF(ENABLE_SPARKLE) IF(ENABLE_SPARKLE)
TARGET_LINK_LIBRARIES(${PROJ_NAME} ${SPARKLE_FRAMEWORK}) TARGET_LINK_LIBRARIES(${PROJ_NAME} ${SPARKLE_FRAMEWORK})
ENDIF(ENABLE_SPARKLE) ENDIF(ENABLE_SPARKLE)
...@@ -448,6 +469,8 @@ SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework Quartz") ...@@ -448,6 +469,8 @@ SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework Quartz")
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework AVFoundation") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework AVFoundation")
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework AddressBook") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework AddressBook")
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework SystemConfiguration") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework SystemConfiguration")
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework MetalKit")
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -framework Metal")
# These variables are specific to our plist and are NOT standard CMake variables # These variables are specific to our plist and are NOT standard CMake variables
SET(MACOSX_BUNDLE_NSMAIN_NIB_FILE "MainMenu") SET(MACOSX_BUNDLE_NSMAIN_NIB_FILE "MainMenu")
......
xcrun -sdk macosx metal -c ../src/Shader.metal -o ../Shader.air
xcrun -sdk macosx metallib ../Shader.air -o ../Shader.metallib
...@@ -17,6 +17,10 @@ ...@@ -17,6 +17,10 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/ */
#import "CurrentCallVC.h" #import "CurrentCallVC.h"
extern "C" {
#import "libavutil/frame.h"
#import "libavutil/display.h"
}
#import <QuartzCore/QuartzCore.h> #import <QuartzCore/QuartzCore.h>
...@@ -41,8 +45,8 @@ ...@@ -41,8 +45,8 @@
#import "delegates/ImageManipulationDelegate.h" #import "delegates/ImageManipulationDelegate.h"
#import "ChatVC.h" #import "ChatVC.h"
#import "views/IconButton.h" #import "views/IconButton.h"
#import "views/CallLayer.h"
#import "utils.h" #import "utils.h"
#import "views/CallMTKView.h"
@interface RendererConnectionsHolder : NSObject @interface RendererConnectionsHolder : NSObject
...@@ -120,7 +124,9 @@ ...@@ -120,7 +124,9 @@
// Video // Video
@property (unsafe_unretained) IBOutlet CallView *videoView; @property (unsafe_unretained) IBOutlet CallView *videoView;
@property (unsafe_unretained) IBOutlet NSView *previewView; @property (unsafe_unretained) IBOutlet CallMTKView *previewView;
@property (unsafe_unretained) IBOutlet CallMTKView *videoMTKView;
@property RendererConnectionsHolder* previewHolder; @property RendererConnectionsHolder* previewHolder;
@property RendererConnectionsHolder* videoHolder; @property RendererConnectionsHolder* videoHolder;
...@@ -129,6 +135,7 @@ ...@@ -129,6 +135,7 @@
@property QMetaObject::Connection messageConnection; @property QMetaObject::Connection messageConnection;
@property QMetaObject::Connection mediaAddedConnection; @property QMetaObject::Connection mediaAddedConnection;
@property QMetaObject::Connection profileUpdatedConnection; @property QMetaObject::Connection profileUpdatedConnection;
@property NSImageView *testView;
@end @end
...@@ -142,6 +149,10 @@ ...@@ -142,6 +149,10 @@
@synthesize previewHolder; @synthesize previewHolder;
@synthesize videoHolder; @synthesize videoHolder;
CVPixelBufferPoolRef pixelBufferPoolDistantView;
CVPixelBufferRef pixelBufferDistantView;
CVPixelBufferPoolRef pixelBufferPoolPreview;
CVPixelBufferRef pixelBufferPreview;
-(void) setCurrentCall:(const std::string&)callUid -(void) setCurrentCall:(const std::string&)callUid
conversation:(const std::string&)convUid conversation:(const std::string&)convUid
...@@ -167,35 +178,9 @@ ...@@ -167,35 +178,9 @@
videoView.callId = callUid; videoView.callId = callUid;
} }
- (void) ensureLayoutForCallStatus:(lrc::api::call::Status) status {
using Status = lrc::api::call::Status;
switch (status) {
case Status::IN_PROGRESS:
if (![videoView.layer isKindOfClass:[CallLayer class]]) {
[videoView setLayer:[[CallLayer alloc] init]];
}
break;
default:
if ([videoView.layer isKindOfClass:[CallLayer class]]) {
[videoView setLayer:[CALayer layer]];
[videoView.layer setBackgroundColor:[[NSColor blackColor] CGColor]];
}
break;
}
holdOnOffButton.image = status == lrc::api::call::Status::PAUSED ?
[NSImage imageNamed:@"ic_action_holdoff.png"] : [NSImage imageNamed:@"ic_action_hold.png"];
}
- (void)awakeFromNib - (void)awakeFromNib
{ {
NSLog(@"INIT CurrentCall VC");
[self.view setWantsLayer:YES]; [self.view setWantsLayer:YES];
[previewView setWantsLayer:YES];
[previewView.layer setBackgroundColor:[NSColor blackColor].CGColor];
[previewView.layer setContentsGravity:kCAGravityResizeAspectFill];
[previewView.layer setFrame:previewView.frame];
[controlsPanel setWantsLayer:YES]; [controlsPanel setWantsLayer:YES];
[controlsPanel.layer setBackgroundColor:[NSColor clearColor].CGColor]; [controlsPanel.layer setBackgroundColor:[NSColor clearColor].CGColor];
[controlsPanel.layer setFrame:controlsPanel.frame]; [controlsPanel.layer setFrame:controlsPanel.frame];
...@@ -294,7 +279,8 @@ ...@@ -294,7 +279,8 @@
[self setBackground]; [self setBackground];
using Status = lrc::api::call::Status; using Status = lrc::api::call::Status;
[self ensureLayoutForCallStatus:currentCall.status]; holdOnOffButton.image = currentCall.status == lrc::api::call::Status::PAUSED ?
[NSImage imageNamed:@"ic_action_holdoff.png"] : [NSImage imageNamed:@"ic_action_hold.png"];
switch (currentCall.status) { switch (currentCall.status) {
case Status::SEARCHING: case Status::SEARCHING:
case Status::CONNECTING: case Status::CONNECTING:
...@@ -321,9 +307,29 @@ ...@@ -321,9 +307,29 @@
[self setupConference:currentCall]; [self setupConference:currentCall];
break;*/ break;*/
case Status::PAUSED: case Status::PAUSED:
[self.videoMTKView fillWithBlack];
[self.previewView fillWithBlack];
[bluerBackgroundEffect setHidden:NO];
[backgroundImage setHidden:NO];
[self.previewView setHidden: YES];
[self.videoMTKView setHidden: YES];
self.previewView.stopRendering = true;
self.videoMTKView.stopRendering = true;
break;
case Status::INACTIVE: case Status::INACTIVE:
if(currentCall.isAudioOnly) {
[self setUpAudioOnlyView];
} else {
[self setUpVideoCallView];
}
break;
case Status::IN_PROGRESS: case Status::IN_PROGRESS:
// change constraints (uncollapse avatar) self.previewView.stopRendering = false;
self.videoMTKView.stopRendering = false;
[previewView fillWithBlack];
[self.videoMTKView fillWithBlack];
[self.previewView setHidden: NO];
[self.videoMTKView setHidden: NO];
if(currentCall.isAudioOnly) { if(currentCall.isAudioOnly) {
[self setUpAudioOnlyView]; [self setUpAudioOnlyView];
} else { } else {
...@@ -345,13 +351,17 @@ ...@@ -345,13 +351,17 @@
[outgoingPhoto setHidden:YES]; [outgoingPhoto setHidden:YES];
[headerContainer setHidden:NO]; [headerContainer setHidden:NO];
[previewView setHidden: NO]; [previewView setHidden: NO];
[self.videoMTKView setHidden:NO];
[bluerBackgroundEffect setHidden:YES]; [bluerBackgroundEffect setHidden:YES];
[backgroundImage setHidden:YES]; [backgroundImage setHidden:YES];
[audioCallView setHidden:YES];
} }
-(void) setUpAudioOnlyView { -(void) setUpAudioOnlyView {
[audioCallView setHidden:NO]; [audioCallView setHidden:NO];
[headerContainer setHidden:YES]; [headerContainer setHidden:YES];
[self.previewView setHidden: YES];
[self.videoMTKView setHidden: YES];
[audioCallPhoto setImage: [self getContactImageOfSize:120.0 withDefaultAvatar:YES]]; [audioCallPhoto setImage: [self getContactImageOfSize:120.0 withDefaultAvatar:YES]];
} }
...@@ -397,7 +407,6 @@ ...@@ -397,7 +407,6 @@
return [[NSImage alloc] initWithData:imageData]; return [[NSImage alloc] initWithData:imageData];
} }
-(void) setupContactInfo:(NSImageView*)imageView -(void) setupContactInfo:(NSImageView*)imageView
{ {
[imageView setImage: [self getContactImageOfSize:120.0 withDefaultAvatar:YES]]; [imageView setImage: [self getContactImageOfSize:120.0 withDefaultAvatar:YES]];
...@@ -426,7 +435,6 @@ ...@@ -426,7 +435,6 @@
} }
} }
-(void) setupConference:(Call*) c -(void) setupConference:(Call*) c
{ {
[videoView setShouldAcceptInteractions:YES]; [videoView setShouldAcceptInteractions:YES];
...@@ -472,8 +480,8 @@ ...@@ -472,8 +480,8 @@
self.videoStarted = QObject::connect(callModel, self.videoStarted = QObject::connect(callModel,
&lrc::api::NewCallModel::remotePreviewStarted, &lrc::api::NewCallModel::remotePreviewStarted,
[self](const std::string& callId, Video::Renderer* renderer) { [self](const std::string& callId, Video::Renderer* renderer) {
[videoView setLayer:[[CallLayer alloc] init]];
[videoView setShouldAcceptInteractions:YES]; [videoView setShouldAcceptInteractions:YES];
[self.videoMTKView setHidden:NO];
[self mouseIsMoving: NO]; [self mouseIsMoving: NO];
[self connectVideoRenderer:renderer]; [self connectVideoRenderer:renderer];
}); });
...@@ -481,11 +489,11 @@ ...@@ -481,11 +489,11 @@
if (callModel->hasCall(callUid_)) { if (callModel->hasCall(callUid_)) {
if (auto renderer = callModel->getRenderer(callUid_)) { if (auto renderer = callModel->getRenderer(callUid_)) {
QObject::disconnect(self.videoStarted); QObject::disconnect(self.videoStarted);
[self.videoMTKView setHidden:NO];
[self connectVideoRenderer: renderer]; [self connectVideoRenderer: renderer];
} }
} }
[self connectPreviewRenderer]; [self connectPreviewRenderer];
} }
-(void) connectPreviewRenderer -(void) connectPreviewRenderer
...@@ -493,31 +501,42 @@ ...@@ -493,31 +501,42 @@
QObject::disconnect(previewHolder.frameUpdated); QObject::disconnect(previewHolder.frameUpdated);
QObject::disconnect(previewHolder.stopped); QObject::disconnect(previewHolder.stopped);
QObject::disconnect(previewHolder.started); QObject::disconnect(previewHolder.started);
previewHolder.started = QObject::connect(&Video::PreviewManager::instance(), previewHolder.started =
QObject::connect(&Video::PreviewManager::instance(),
&Video::PreviewManager::previewStarted, &Video::PreviewManager::previewStarted,
[=](Video::Renderer* renderer) { [=](Video::Renderer* renderer) {
[self.previewView setHidden:NO];
self.previewView.stopRendering = false;
QObject::disconnect(previewHolder.frameUpdated); QObject::disconnect(previewHolder.frameUpdated);
previewHolder.frameUpdated = QObject::connect(renderer, previewHolder.frameUpdated =
&Video::Renderer::frameUpdated, QObject::connect(renderer,
[=]() { &Video::Renderer::frameUpdated,
[self renderer:Video::PreviewManager::instance().previewRenderer() [=]() {
renderFrameForPreviewView:previewView]; if(!renderer->isRendering()) {
}); return;
}); }
[self renderer:renderer renderFrameForPreviewView: self.previewView];
});
});
previewHolder.stopped = QObject::connect(&Video::PreviewManager::instance(), previewHolder.stopped = QObject::connect(&Video::PreviewManager::instance(),
&Video::PreviewManager::previewStopped, &Video::PreviewManager::previewStopped,
[=](Video::Renderer* renderer) { [=](Video::Renderer* renderer) {
QObject::disconnect(previewHolder.frameUpdated); QObject::disconnect(previewHolder.frameUpdated);
[previewView.layer setContents:nil]; [self.previewView setHidden:YES];
self.previewView.stopRendering = true;
}); });
previewHolder.frameUpdated = QObject::connect(Video::PreviewManager::instance().previewRenderer(), previewHolder.frameUpdated =
&Video::Renderer::frameUpdated, QObject::connect(Video::PreviewManager::instance().previewRenderer(),
[=]() { &Video::Renderer::frameUpdated,
[self renderer:Video::PreviewManager::instance().previewRenderer() [=]() {
renderFrameForPreviewView:previewView]; if(!Video::PreviewManager::instance()
}); .previewRenderer()->isRendering()) {
return;
}
[self renderer:Video::PreviewManager::instance()
.previewRenderer() renderFrameForPreviewView: self.previewView];
});
} }
-(void) connectVideoRenderer: (Video::Renderer*)renderer -(void) connectVideoRenderer: (Video::Renderer*)renderer
...@@ -535,80 +554,222 @@ ...@@ -535,80 +554,222 @@
if(!renderer->isRendering()) { if(!renderer->isRendering()) {
return; return;
} }
[self renderer:renderer renderFrameForDistantView:videoView]; [self renderer:renderer renderFrameForDistantView:self.videoMTKView];
}); });
videoHolder.started = QObject::connect(renderer, videoHolder.started =
QObject::connect(renderer,
&Video::Renderer::started, &Video::Renderer::started,
[=]() { [=]() {
if (![videoView.layer isKindOfClass:[CallLayer class]]) {
[videoView setLayer:[[CallLayer alloc] init]];
}
[self mouseIsMoving: NO]; [self mouseIsMoving: NO];
self.videoMTKView.stopRendering = false;
[self.videoMTKView setHidden:NO];
[bluerBackgroundEffect setHidden:YES];
[backgroundImage setHidden:YES];
[videoView setShouldAcceptInteractions:YES]; [videoView setShouldAcceptInteractions:YES];
QObject::disconnect(videoHolder.frameUpdated); QObject::disconnect(videoHolder.frameUpdated);
videoHolder.frameUpdated = QObject::connect(renderer, videoHolder.frameUpdated
&Video::Renderer::frameUpdated, = QObject::connect(renderer,
[=]() { &Video::Renderer::frameUpdated,
[self renderer:renderer renderFrameForDistantView:videoView]; [=]() {
}); if(!renderer->isRendering()) {
return;
}
[self renderer:renderer renderFrameForDistantView:self.videoMTKView];
});
}); });
videoHolder.stopped = QObject::connect(renderer, videoHolder.stopped = QObject::connect(renderer,
&Video::Renderer::stopped, &Video::Renderer::stopped,
[=]() { [=]() {
[(CallLayer*)videoView.layer setVideoRunning:NO];
[videoView setLayer:[CALayer layer]];
[videoView.layer setBackgroundColor:[[NSColor blackColor] CGColor]];
[self mouseIsMoving: YES]; [self mouseIsMoving: YES];
self.videoMTKView.stopRendering = true;
[self.videoMTKView setHidden:YES];
[bluerBackgroundEffect setHidden:NO];
[backgroundImage setHidden:NO];
[videoView setShouldAcceptInteractions:NO]; [videoView setShouldAcceptInteractions:NO];
QObject::disconnect(videoHolder.frameUpdated); QObject::disconnect(videoHolder.frameUpdated);
}); });
} }
-(void) renderer: (Video::Renderer*)renderer renderFrameForPreviewView:(NSView*) view -(void) renderer: (Video::Renderer*)renderer renderFrameForPreviewView:(CallMTKView*) view
{ {
QSize res = renderer->size(); @autoreleasepool {
auto framePtr = renderer->currentAVFrame();
auto frame_ptr = renderer->currentFrame(); auto frame = framePtr.get();
auto frame_data = frame_ptr.ptr; if(!frame || !frame->width || !frame->height) {
if (!frame_data) return;
return; }
auto frameSize = CGSizeMake(frame->width, frame->height);
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); auto rotation = 0;
CGContextRef newContext = CGBitmapContextCreate(frame_data, if (frame->data[3] != NULL && (CVPixelBufferRef)frame->data[3]) {
res.width(), [view renderWithPixelBuffer:(CVPixelBufferRef)frame->data[3]
res.height(), size: frameSize
8, rotation: rotation
4*res.width(), fillFrame: true];
colorSpace, return;
kCGImageAlphaPremultipliedLast); }
else if (CVPixelBufferRef pixelBuffer = [self getBufferForPreviewFromFrame:frame]) {
[view renderWithPixelBuffer: pixelBuffer
CGImageRef newImage = CGBitmapContextCreateImage(newContext); size: frameSize
rotation: rotation
/*We release some components*/ fillFrame: true];
CGContextRelease(newContext); }
CGColorSpaceRelease(colorSpace); }
[CATransaction begin];
view.layer.contents = (__bridge id)newImage;
[CATransaction commit];
CFRelease(newImage);
} }
-(void) renderer: (Video::Renderer*)renderer renderFrameForDistantView:(CallView*) view -(void) renderer: (Video::Renderer*)renderer renderFrameForDistantView:(CallMTKView*) view
{ {
auto frame_ptr = renderer->currentFrame(); @autoreleasepool {
if (!frame_ptr.ptr) auto framePtr = renderer->currentAVFrame();
return; auto frame = framePtr.get();
if(!frame || !frame->width || !frame->height) {
return;
}
auto frameSize = CGSizeMake(frame->width, frame->height);
auto rotation = 0;
if (auto matrix = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX)) {
const int32_t* data = reinterpret_cast<int32_t*>(matrix->data);
rotation = av_display_rotation_get(data);
}
if (frame->data[3] != NULL && (CVPixelBufferRef)frame->data[3]) {
[view renderWithPixelBuffer: (CVPixelBufferRef)frame->data[3]
size: frameSize
rotation: rotation
fillFrame: false];
}
if (CVPixelBufferRef pixelBuffer = [self getBufferForDistantViewFromFrame:frame]) {
[view renderWithPixelBuffer: pixelBuffer
size: frameSize
rotation: rotation
fillFrame: false];
}
}
}
-(CVPixelBufferRef) getBufferForPreviewFromFrame:(const AVFrame*)frame {
if(!frame || !frame->data[0] || !frame->data[1]) {
return nil;
}
CVReturn theError;
bool createPool = false;
if (!pixelBufferPoolPreview) {
createPool = true;
} else {
NSDictionary* atributes = (__bridge NSDictionary*)CVPixelBufferPoolGetAttributes(pixelBufferPoolPreview);
int width = [[atributes objectForKey:(NSString*)kCVPixelBufferWidthKey] intValue];
int height = [[atributes objectForKey:(NSString*)kCVPixelBufferHeightKey] intValue];
if (width != frame->width || height != frame->height) {
createPool = true;
}
}
if (createPool) {
CVPixelBufferPoolRelease(pixelBufferPoolPreview);
CVPixelBufferRelease(pixelBufferPreview);
pixelBufferPreview = nil;
pixelBufferPoolPreview = nil;
NSMutableDictionary* attributes = [NSMutableDictionary dictionary];
[attributes setObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] forKey:(NSString*)kCVPixelBufferPixelFormatTypeKey];
[attributes setObject:[NSNumber numberWithInt:frame->width] forKey: (NSString*)kCVPixelBufferWidthKey];
[attributes setObject:[NSNumber numberWithInt:frame->height] forKey: (NSString*)kCVPixelBufferHeightKey];
[attributes setObject:@(frame->linesize[0]) forKey:(NSString*)kCVPixelBufferBytesPerRowAlignmentKey];
[attributes setObject:[NSDictionary dictionary] forKey:(NSString*)kCVPixelBufferIOSurfacePropertiesKey];
theError = CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL, (__bridge CFDictionaryRef) attributes, &pixelBufferPoolPreview);
if (theError != kCVReturnSuccess) {
NSLog(@"CVPixelBufferPoolCreate Failed");
return nil;
}
}
if(!pixelBufferPreview) {
theError = CVPixelBufferPoolCreatePixelBuffer(NULL, pixelBufferPoolPreview, &pixelBufferPreview);
if(theError != kCVReturnSuccess) {
NSLog(@"CVPixelBufferPoolCreatePixelBuffer Failed");
return nil;
}
}
theError = CVPixelBufferLockBaseAddress(pixelBufferPreview, 0);
if (theError != kCVReturnSuccess) {
NSLog(@"lock error");
return nil;
}
size_t bytePerRowY = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferPreview, 0);
size_t bytesPerRowUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferPreview, 1);
void* base = CVPixelBufferGetBaseAddressOfPlane(pixelBufferPreview, 0);
memcpy(base, frame->data[0], bytePerRowY * frame->height);
base = CVPixelBufferGetBaseAddressOfPlane(pixelBufferPreview, 1);
memcpy(base, frame->data[1], bytesPerRowUV * frame->height/2);
CVPixelBufferUnlockBaseAddress(pixelBufferPreview, 0);
return pixelBufferPreview;
}
CallLayer* callLayer = (CallLayer*) view.layer; -(CVPixelBufferRef) getBufferForDistantViewFromFrame:(const AVFrame*)frame {
if ([callLayer respondsToSelector:@selector(setCurrentFrame:)]) { if(!frame || !frame->data[0] || !frame->data[1]) {
[callLayer setCurrentFrame:std::move(frame_ptr)]; return nil;
[callLayer setVideoRunning:YES]; }
CVReturn theError;
bool createPool = false;
if (!pixelBufferPoolDistantView){
createPool = true;
}
NSDictionary* atributes = (__bridge NSDictionary*)CVPixelBufferPoolGetPixelBufferAttributes(pixelBufferPoolDistantView);
int width = [[atributes objectForKey:(NSString*)kCVPixelBufferWidthKey] intValue];
int height = [[atributes objectForKey:(NSString*)kCVPixelBufferHeightKey] intValue];
if (width != frame->width || height != frame->height) {
createPool = true;
} }
if (createPool) {
CVPixelBufferPoolRelease(pixelBufferPoolDistantView);
CVPixelBufferRelease(pixelBufferDistantView);
pixelBufferDistantView = nil;
pixelBufferPoolDistantView = nil;
NSMutableDictionary* attributes = [NSMutableDictionary dictionary];
[attributes setObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] forKey:(NSString*)kCVPixelBufferPixelFormatTypeKey];
[attributes setObject:[NSNumber numberWithInt:frame->width] forKey: (NSString*)kCVPixelBufferWidthKey];
[attributes setObject:[NSNumber numberWithInt:frame->height] forKey: (NSString*)kCVPixelBufferHeightKey];
[attributes setObject:@(frame->linesize[0]) forKey:(NSString*)kCVPixelBufferBytesPerRowAlignmentKey];
[attributes setObject:[NSDictionary dictionary] forKey:(NSString*)kCVPixelBufferIOSurfacePropertiesKey];
theError = CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL, (__bridge CFDictionaryRef) attributes, &pixelBufferPoolDistantView);
if (theError != kCVReturnSuccess) {
return nil;
NSLog(@"CVPixelBufferPoolCreate Failed");
}
}
if(!pixelBufferDistantView) {
theError = CVPixelBufferPoolCreatePixelBuffer(NULL, pixelBufferPoolDistantView, &pixelBufferDistantView);
if(theError != kCVReturnSuccess) {
return nil;
NSLog(@"CVPixelBufferPoolCreatePixelBuffer Failed");
}
}
theError = CVPixelBufferLockBaseAddress(pixelBufferDistantView, 0);
if (theError != kCVReturnSuccess) {
return nil;
NSLog(@"lock error");
}
size_t bytePerRowY = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferDistantView, 0);
size_t bytesPerRowUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBufferDistantView, 1);
void* base = CVPixelBufferGetBaseAddressOfPlane(pixelBufferDistantView, 0);
memcpy(base, frame->data[0], bytePerRowY * frame->height);
base = CVPixelBufferGetBaseAddressOfPlane(pixelBufferDistantView, 1);
uint32_t size = frame->linesize[1] * frame->height / 2;
uint8_t* dstData = new uint8_t[2 * size];
uint8_t * firstData = new uint8_t[size];
memcpy(firstData, frame->data[1], size);
uint8_t * secondData = new uint8_t[size];
memcpy(secondData, frame->data[2], size);
for (int i = 0; i < 2 * size; i++){
if (i % 2 == 0){
dstData[i] = firstData[i/2];
}else {
dstData[i] = secondData[i/2];
}
}
memcpy(base, dstData, bytesPerRowUV * frame->height/2);
CVPixelBufferUnlockBaseAddress(pixelBufferDistantView, 0);
free(dstData);
free(firstData);
free(secondData);
return pixelBufferDistantView;
} }
- (void) initFrame - (void) initFrame
...@@ -617,6 +778,8 @@ ...@@ -617,6 +778,8 @@
[self.view setHidden:YES]; [self.view setHidden:YES];
self.view.layer.position = self.view.frame.origin; self.view.layer.position = self.view.frame.origin;
[self collapseRightView]; [self collapseRightView];
self.testView = [[NSImageView alloc] initWithFrame:self.view.frame];
[self.view addSubview:self.testView];
} }
# pragma private IN/OUT animations # pragma private IN/OUT animations
...@@ -653,10 +816,6 @@ ...@@ -653,10 +816,6 @@
QObject::disconnect(previewHolder.stopped); QObject::disconnect(previewHolder.stopped);
QObject::disconnect(previewHolder.started); QObject::disconnect(previewHolder.started);
QObject::disconnect(self.messageConnection); QObject::disconnect(self.messageConnection);
[previewView.layer setContents:nil];
[previewView setHidden: YES];
[videoView setLayer:[CALayer layer]];
[videoView.layer setBackgroundColor:[[NSColor blackColor] CGColor]];
[self.chatButton setHidden:YES]; [self.chatButton setHidden:YES];
[self.chatButton setPressed:NO]; [self.chatButton setPressed:NO];
...@@ -677,6 +836,8 @@ ...@@ -677,6 +836,8 @@
//outgoing view //outgoing view
[outgoingPersonLabel setStringValue:@""]; [outgoingPersonLabel setStringValue:@""];
[outgoingStateLabel setStringValue:@""]; [outgoingStateLabel setStringValue:@""];
[self.previewView setHidden:YES];
[self.videoMTKView setHidden:YES];
} }
-(void) setupCallView -(void) setupCallView
...@@ -815,7 +976,6 @@ ...@@ -815,7 +976,6 @@
return; return;
auto* callModel = accountInfo_->callModel.get(); auto* callModel = accountInfo_->callModel.get();
callModel->hangUp(callUid_); callModel->hangUp(callUid_);
} }
......
...@@ -37,6 +37,7 @@ ...@@ -37,6 +37,7 @@
#import <api/contact.h> #import <api/contact.h>
#import <api/datatransfermodel.h> #import <api/datatransfermodel.h>
#import <media/recordingmodel.h> #import <media/recordingmodel.h>
#import <api/avmodel.h>