Commit 14e7bf33 authored by Anthony Léonard's avatar Anthony Léonard

video: OpenGL rendering of incoming frames

The previous system to display the incoming video frames was relying
on CGImage generation with raw framebuffer which were then set as the
content of the CallView. This way of doing it is not efficient as it
implies buffer copies and is discouraged by Apple for pictures that
change often. Moreover, this process was done by the
VideoReceiveThread from the daemon which was then blocked by those
copies without being able to decode further incoming frames. This is
why a lag was appearing and increasing on high resolution stream.

The new system now isolates frame delivering to the UI and their
rendering. The VideoReceiveThread just update the current frame buffer
and size without copy and another thread send those data to an OpenGL
texture on screen refresh which also enables to automatically skip
frames in case of heavy load.

Change-Id: I0b79ddce66f52a3db1eee19945733ff93e7ce34f
Reviewed-by: Kateryna Kostiuk's avatarKateryna Kostiuk <kateryna.kostiuk@savoirfairelinux.com>
parent d2f7b5d0
......@@ -24,6 +24,7 @@ FIND_PACKAGE(Qt5Core REQUIRED)
FIND_PACKAGE(Qt5MacExtras REQUIRED)
FIND_PACKAGE(Qt5Widgets REQUIRED)
FIND_PACKAGE(LibRingClient REQUIRED)
FIND_PACKAGE(OpenGL REQUIRED)
EXECUTE_PROCESS(COMMAND git submodule update --init
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
......@@ -79,6 +80,7 @@ INCLUDE_DIRECTORIES(SYSTEM ${Qt5MacExtras_INCLUDE_DIRS})
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR})
INCLUDE_DIRECTORIES( ${CMAKE_CURRENT_SOURCE_DIR})
INCLUDE_DIRECTORIES(${LIB_RING_CLIENT_INCLUDE_DIR})
INCLUDE_DIRECTORIES(${OPENGL_INCLUDE_DIR})
SET(CMAKE_MACOSX_RPATH ON)
SET(CMAKE_SKIP_BUILD_RPATH FALSE)
......@@ -179,6 +181,8 @@ SET(ringclient_BACKENDS
SET(ringclient_VIEWS
src/views/CallView.mm
src/views/CallView.h
src/views/CallLayer.mm
src/views/CallLayer.h
src/views/ITProgressIndicator.mm
src/views/ITProgressIndicator.h
src/views/RingOutlineView.mm
......@@ -420,6 +424,7 @@ TARGET_LINK_LIBRARIES( ${PROJ_NAME}
${Qt5Core_LIBRARIES}
${Qt5MacExtras_LIBRARIES}
${Qt5Widgets_LIBRARIES}
${OPENGL_LIBRARIES}
-lqrencode
)
......
......@@ -48,6 +48,7 @@
#import "ChatVC.h"
#import "BrokerVC.h"
#import "views/IconButton.h"
#import "views/CallLayer.h"
@interface RendererConnectionsHolder : NSObject
......@@ -255,11 +256,6 @@
actionHash[ (int)UserActionModel::Action::MUTE_AUDIO] = muteAudioButton;
actionHash[ (int)UserActionModel::Action::MUTE_VIDEO] = muteVideoButton;
[videoView setWantsLayer:YES];
[videoView.layer setBackgroundColor:[NSColor blackColor].CGColor];
[videoView.layer setFrame:videoView.frame];
[videoView.layer setContentsGravity:kCAGravityResizeAspect];
[previewView setWantsLayer:YES];
[previewView.layer setBackgroundColor:[NSColor blackColor].CGColor];
[previewView.layer setContentsGravity:kCAGravityResizeAspectFill];
......@@ -392,7 +388,7 @@
&Video::Renderer::frameUpdated,
[=]() {
[self renderer:Video::PreviewManager::instance().previewRenderer()
renderFrameForView:previewView];
renderFrameForPreviewView:previewView];
});
});
......@@ -407,7 +403,7 @@
&Video::Renderer::frameUpdated,
[=]() {
[self renderer:Video::PreviewManager::instance().previewRenderer()
renderFrameForView:previewView];
renderFrameForPreviewView:previewView];
});
}
......@@ -419,7 +415,7 @@
videoHolder.frameUpdated = QObject::connect(renderer,
&Video::Renderer::frameUpdated,
[=]() {
[self renderer:renderer renderFrameForView:videoView];
[self renderer:renderer renderFrameForDistantView:videoView];
});
videoHolder.started = QObject::connect(renderer,
......@@ -429,7 +425,7 @@
videoHolder.frameUpdated = QObject::connect(renderer,
&Video::Renderer::frameUpdated,
[=]() {
[self renderer:renderer renderFrameForView:videoView];
[self renderer:renderer renderFrameForDistantView:videoView];
});
});
......@@ -437,11 +433,11 @@
&Video::Renderer::stopped,
[=]() {
QObject::disconnect(videoHolder.frameUpdated);
[videoView.layer setContents:nil];
[(CallLayer*)videoView.layer setVideoRunning:NO];
});
}
-(void) renderer: (Video::Renderer*)renderer renderFrameForView:(NSView*) view
-(void) renderer: (Video::Renderer*)renderer renderFrameForPreviewView:(NSView*) view
{
QSize res = renderer->size();
......@@ -450,7 +446,6 @@
if (!frame_data)
return;
CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
CGContextRef newContext = CGBitmapContextCreate(frame_data,
res.width(),
......@@ -474,6 +469,20 @@
CFRelease(newImage);
}
-(void) renderer: (Video::Renderer*)renderer renderFrameForDistantView:(CallView*) view
{
QSize res = renderer->size();
auto frame_ptr = renderer->currentFrame();
if (!frame_ptr.ptr)
return;
CallLayer* callLayer = (CallLayer*) view.layer;
[callLayer setCurrentFrame:std::move(frame_ptr) ofSize:res];
[callLayer setVideoRunning:YES];
}
- (void) initFrame
{
[self.view setFrame:self.view.superview.bounds];
......@@ -538,7 +547,6 @@
QObject::disconnect(previewHolder.frameUpdated);
QObject::disconnect(previewHolder.stopped);
QObject::disconnect(previewHolder.started);
[videoView.layer setContents:nil];
[previewView.layer setContents:nil];
[_brokerPopoverVC performClose:self];
......
/*
* Copyright (C) 2017 Savoir-faire Linux Inc.
* Author: Anthony Léonard <anthony.leonard@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#import <Cocoa/Cocoa.h>
#import <QSize>
#import <video/renderer.h>
@interface CallLayer : NSOpenGLLayer
@property BOOL videoRunning;
- (void) setCurrentFrame:(Video::Frame)framePtr ofSize:(QSize)frameSize;
@end
/*
* Copyright (C) 2017 Savoir-faire Linux Inc.
* Author: Anthony Léonard <anthony.leonard@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#import "CallLayer.h"
#import <OpenGL/gl3.h>
static const GLchar* vShaderSrc = R"glsl(
#version 150
in vec2 in_Pos;
in vec2 in_TexCoord;
uniform vec2 in_Scaling;
out vec2 texCoord;
void main()
{
texCoord = in_TexCoord;
gl_Position = vec4(in_Pos.x*in_Scaling.x, in_Pos.y*in_Scaling.y, 0.0, 1.0);
}
)glsl";
static const GLchar* fShaderSrc = R"glsl(
#version 150
out vec4 fragColor;
in vec2 texCoord;
uniform sampler2D tex;
void main()
{
fragColor = texture(tex, texCoord);
}
)glsl";
@implementation CallLayer
// OpenGL handlers
GLuint tex, vbo, vShader, fShader, sProg, vao;
// Last frame data and attributes
Video::Frame currentFrame;
QSize currentFrameSize;
BOOL currentFrameDisplayed;
NSLock* currentFrameLk;
- (id) init
{
self = [super init];
if (self) {
currentFrameLk = [[NSLock alloc] init];
[self setVideoRunning:NO];
}
return self;
}
// This setter is redefined so we can initialize the OpenGL context when this one is
// setup by the UI (which seems to be done just before the first draw attempt and not in init method);
- (void)setOpenGLContext:(NSOpenGLContext *)openGLContext
{
[super setOpenGLContext:openGLContext];
if (openGLContext) {
GLfloat vertices[] = {
-1.0, 1.0, 0.0, 0.0, // Top-left
1.0, 1.0, 1.0, 0.0, // Top-right
-1.0, -1.0, 0.0, 1.0, // Bottom-left
1.0, -1.0, 1.0, 1.0 // Bottom-right
};
[openGLContext makeCurrentContext];
// VAO
glGenVertexArrays(1, &vao);
glBindVertexArray(vao);
// VBO
glGenBuffers(1, &vbo);
glBindBuffer(GL_ARRAY_BUFFER, vbo);
glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
// Vertex shader
vShader = glCreateShader(GL_VERTEX_SHADER);
glShaderSource(vShader, 1, &vShaderSrc, NULL);
glCompileShader(vShader);
// Fragment shader
fShader = glCreateShader(GL_FRAGMENT_SHADER);
glShaderSource(fShader, 1, &fShaderSrc, NULL);
glCompileShader(fShader);
// Program
sProg = glCreateProgram();
glAttachShader(sProg, vShader);
glAttachShader(sProg, fShader);
glBindFragDataLocation(sProg, 0, "fragColor");
glLinkProgram(sProg);
glUseProgram(sProg);
// Vertices position attrib
GLuint inPosAttrib = glGetAttribLocation(sProg, "in_Pos");
glEnableVertexAttribArray(inPosAttrib);
glVertexAttribPointer(inPosAttrib, 2, GL_FLOAT, GL_FALSE, 4*sizeof(GLfloat), 0);
// Texture position attrib
GLuint inTexCoordAttrib = glGetAttribLocation(sProg, "in_TexCoord");
glEnableVertexAttribArray(inTexCoordAttrib);
glVertexAttribPointer(inTexCoordAttrib, 2, GL_FLOAT, GL_FALSE, 4*sizeof(GLfloat), (void*)(2*sizeof(GLfloat)));
// Texture
glGenTextures(1, &tex);
glBindTexture(GL_TEXTURE_2D, tex);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_BORDER);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_BORDER);
}
}
- (NSOpenGLPixelFormat *)openGLPixelFormatForDisplayMask:(uint32_t)mask
{
NSOpenGLPixelFormatAttribute attrs[] = {
NSOpenGLPFANoRecovery,
NSOpenGLPFAColorSize, 24,
NSOpenGLPFAAlphaSize, 8,
NSOpenGLPFADoubleBuffer,
NSOpenGLPFAScreenMask,
mask,
NSOpenGLPFAAccelerated,
NSOpenGLPFAOpenGLProfile,
NSOpenGLProfileVersion3_2Core,
0
};
NSOpenGLPixelFormat* pixelFormat = [[NSOpenGLPixelFormat alloc] initWithAttributes:attrs];
return pixelFormat;
}
- (BOOL)isAsynchronous
{
return YES;
}
- (void)drawInOpenGLContext:(NSOpenGLContext *)context pixelFormat:(NSOpenGLPixelFormat *)pixelFormat forLayerTime:(CFTimeInterval)t displayTime:(const CVTimeStamp *)ts
{
GLenum errEnum;
glBindTexture(GL_TEXTURE_2D, tex);
[currentFrameLk lock];
if(!currentFrameDisplayed) {
if(currentFrame.ptr)
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, currentFrameSize.width(), currentFrameSize.height(), 0, GL_RGBA, GL_UNSIGNED_BYTE, currentFrame.ptr);
currentFrameDisplayed = YES;
}
// To ensure that we will not divide by zero
if (!currentFrameSize.isEmpty()) {
// Compute scaling factor to keep the original aspect ratio of the video
CGSize viewSize = self.frame.size;
float viewRatio = viewSize.width/viewSize.height;
float frameRatio = ((float)currentFrameSize.width())/((float)currentFrameSize.height());
float ratio = viewRatio * (1/frameRatio);
GLint inScalingUniform = glGetUniformLocation(sProg, "in_Scaling");
if (ratio < 1.0)
glUniform2f(inScalingUniform, 1.0, ratio);
else
glUniform2f(inScalingUniform, 1.0/ratio, 1.0);
}
[currentFrameLk unlock];
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
if([self videoRunning])
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
}
- (void) setCurrentFrame:(Video::Frame)framePtr ofSize:(QSize)frameSize
{
[currentFrameLk lock];
currentFrame = std::move(framePtr);
currentFrameSize = frameSize;
currentFrameDisplayed = NO;
[currentFrameLk unlock];
}
@end
......@@ -18,6 +18,7 @@
*/
#import "CallView.h"
#import "CallLayer.h"
#import <QItemSelectionModel>
#import <QAbstractProxyModel>
......@@ -49,6 +50,7 @@
if (self)
{
[self registerForDraggedTypes:[NSArray arrayWithObjects:NSFilenamesPboardType, nil]];
[self setWantsLayer:YES];
}
[self.window setAcceptsMouseMovedEvents:YES];
......@@ -64,6 +66,11 @@
return self;
}
- (CALayer *)makeBackingLayer
{
return (CALayer*) [[CallLayer alloc] init];
}
#pragma mark - Destination Operations
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment