From 361cdaf87f5a1158fb3e006bcb0c27e79b76cf55 Mon Sep 17 00:00:00 2001
From: Mathias Paulin <mathias.paulin@irit.fr>
Date: Wed, 14 Apr 2021 12:08:19 +0200
Subject: [PATCH] Prototype an image processing (2D) pass
---
src/libRender/CMakeLists.txt | 2 +
.../RadiumNBR/Passes/ImageProcessPass.cpp | 140 ++++++++++++++++++
.../RadiumNBR/Passes/ImageProcessPass.hpp | 59 ++++++++
.../RadiumNBR/Renderer/Visualization.cpp | 25 +++-
.../RadiumNBR/Renderer/Visualization.hpp | 6 +
.../ImageProcess.frag.glsl | 77 ++++++++++
.../ImageProcess.vert.glsl | 7 +
7 files changed, 315 insertions(+), 1 deletion(-)
create mode 100644 src/libRender/RadiumNBR/Passes/ImageProcessPass.cpp
create mode 100644 src/libRender/RadiumNBR/Passes/ImageProcessPass.hpp
create mode 100644 src/libRender/RadiumNBR/Shaders/ImageProcessingPass/ImageProcess.frag.glsl
create mode 100644 src/libRender/RadiumNBR/Shaders/ImageProcessingPass/ImageProcess.vert.glsl
diff --git a/src/libRender/CMakeLists.txt b/src/libRender/CMakeLists.txt
index fd86e22..4bb346a 100644
--- a/src/libRender/CMakeLists.txt
+++ b/src/libRender/CMakeLists.txt
@@ -41,6 +41,7 @@ set(sources
RadiumNBR/Passes/AccessibilityBufferPass.cpp
RadiumNBR/Passes/EnvLightPass.cpp
RadiumNBR/Passes/EmissivityPass.cpp
+ RadiumNBR/Passes/ImageProcessPass.cpp
RadiumNBR/Passes/LocalLightPass.cpp
RadiumNBR/Passes/TransparencyPass.cpp
RadiumNBR/Passes/UiPass.cpp
@@ -62,6 +63,7 @@ set(public_headers
RadiumNBR/Passes/AccessibilityBufferPass.hpp
RadiumNBR/Passes/EmissivityPass.hpp
RadiumNBR/Passes/EnvLightPass.hpp
+ RadiumNBR/Passes/ImageProcessPass.hpp
RadiumNBR/Passes/LocalLightPass.hpp
RadiumNBR/Passes/TransparencyPass.hpp
RadiumNBR/Passes/UiPass.hpp
diff --git a/src/libRender/RadiumNBR/Passes/ImageProcessPass.cpp b/src/libRender/RadiumNBR/Passes/ImageProcessPass.cpp
new file mode 100644
index 0000000..690b507
--- /dev/null
+++ b/src/libRender/RadiumNBR/Passes/ImageProcessPass.cpp
@@ -0,0 +1,140 @@
+#include <RadiumNBR/Passes/ImageProcessPass.hpp>
+
+#ifdef PASSES_LOG
+# include <Core/Utils/Log.hpp>
+using namespace Ra::Core::Utils; // log
+#endif
+
+#include <Core/Geometry/MeshPrimitives.hpp>
+
+#include <Engine/Data/Mesh.hpp>
+#include <Engine/Data/ShaderConfigFactory.hpp>
+#include <Engine/Data/ShaderProgramManager.hpp>
+#include <Engine/Data/Texture.hpp>
+#include <Engine/Data/ViewingParameters.hpp>
+#include <Engine/RadiumEngine.hpp>
+
+#include <globjects/Framebuffer.h>
+
+namespace RadiumNBR {
+using namespace gl;
+
+static const GLenum buffer = { GL_COLOR_ATTACHMENT0 };
+
+ImageProcessPass::ImageProcessPass( const Ra::Core::Utils::Index& idx ) :
+ RenderPass( "Image processing pass", idx, nullptr ) {}
+
+ImageProcessPass::~ImageProcessPass() = default;
+
+
+bool ImageProcessPass::initializePass( size_t width,
+ size_t height,
+ Ra::Engine::Data::ShaderProgramManager* shaderMngr ) {
+ // Initialize the fbo
+ m_fbo = std::make_unique<globjects::Framebuffer>();
+
+ Ra::Engine::Data::TextureParameters texparams;
+ texparams.width = width;
+ texparams.height = height;
+ texparams.target = GL_TEXTURE_2D;
+ texparams.minFilter = GL_LINEAR;
+ texparams.magFilter = GL_LINEAR;
+ texparams.internalFormat = GL_RGBA32F;
+ texparams.format = GL_RGBA;
+ texparams.type = GL_FLOAT;
+ texparams.name = "ImageProcess::Result";
+ m_sharedTextures.insert(
+ {texparams.name, std::make_shared<Ra::Engine::Data::Texture>( texparams )} );
+
+ // The shader caller
+ Ra::Core::Geometry::TriangleMesh mesh =
+ Ra::Core::Geometry::makeZNormalQuad( Ra::Core::Vector2( -1.f, 1.f ) );
+ auto qm = std::make_unique<Ra::Engine::Data::Mesh>( "caller" );
+ qm->loadGeometry( std::move( mesh ) );
+ m_quadMesh = std::move( qm );
+ m_quadMesh->updateGL();
+
+ // The shader
+ // TODO, based on the same approach than in CustomAttribToColorPass,
+ // in the future, this shader might be user-defined
+ std::string resourcesRootDir = getResourcesDir();
+ auto added = shaderMngr->addShaderProgram(
+ {{"ImageProcess(LIC)"},
+ resourcesRootDir + "Shaders/ImageProcessingPass/ImageProcess.vert.glsl",
+ resourcesRootDir + "Shaders/ImageProcessingPass/ImageProcess.frag.glsl"} );
+ if ( added ) { m_shader = added.value(); }
+ else
+ { return false; }
+
+ return true;
+}
+
+
+bool ImageProcessPass::update() {
+ // nothing to do
+ return true;
+}
+
+void ImageProcessPass::resize( size_t width, size_t height ) {
+ for ( auto& t : m_sharedTextures )
+ {
+ t.second->resize( width, height );
+ }
+
+ m_fbo->bind();
+ m_fbo->attachTexture( GL_COLOR_ATTACHMENT0, m_sharedTextures["ImageProcess::Result"]->texture() );
+#ifdef PASSES_LOG
+ if ( m_fbo->checkStatus() != GL_FRAMEBUFFER_COMPLETE )
+ { LOG( logERROR ) << "FBO Error (GeomPrePass::resize): " << m_fbo->checkStatus(); }
+#endif
+
+ globjects::Framebuffer::unbind();
+}
+
+void ImageProcessPass::execute(
+ const Ra::Engine::Data::ViewingParameters& viewParams ) const {
+ m_fbo->bind();
+ GL_ASSERT( glDrawBuffers( 1, &buffer ) );
+ GL_ASSERT( glClearBufferfv( GL_COLOR, 0, Ra::Core::Utils::Color::White().data() ) );
+ GL_ASSERT( glDisable( GL_DEPTH_TEST ) );
+ GL_ASSERT( glDepthMask( GL_FALSE ) );
+
+ m_shader->bind();
+ Ra::Core::Matrix4 viewproj = viewParams.projMatrix * viewParams.viewMatrix;
+
+ m_shader->setUniform( "transform.mvp", viewproj );
+ m_shader->setUniform( "transform.proj", viewParams.projMatrix );
+ m_shader->setUniform( "transform.view", viewParams.viewMatrix );
+
+ const auto td = m_importedTextures.find( "ImageProcess::DepthSampler" );
+ m_shader->setUniform( "depth_sampler", td->second.get(), 0 );
+ const auto tc = m_importedTextures.find( "ImageProcess::ColorSampler" );
+ m_shader->setUniform( "image_sampler", tc->second.get(), 1 );
+
+ m_quadMesh->render( m_shader );
+ GL_ASSERT( glEnable( GL_DEPTH_TEST ) );
+ GL_ASSERT( glDepthMask( GL_TRUE ) );
+ m_fbo->unbind();
+
+}
+
+/// These inputs must be computed before executing this pass
+/// positions and normals must be computed in world space
+void ImageProcessPass::setInputs( const SharedTextures& depthBuffer,
+ const SharedTextures& colorBuffer ) {
+ m_importedTextures["ImageProcess::DepthSampler"] = depthBuffer.second;
+ m_importedTextures["ImageProcess::ColorSampler"] = colorBuffer.second;
+}
+
+void ImageProcessPass::setOutput( const SharedTextures& colorBuffer ) {
+ m_outputTexture = colorBuffer;
+}
+
+// Nothing to do, this pass is screen space
+bool ImageProcessPass::buildRenderTechnique(
+ const Ra::Engine::Rendering::RenderObject* ro,
+ Ra::Engine::Rendering::RenderTechnique& rt ) const {
+ return true;
+};
+
+}
diff --git a/src/libRender/RadiumNBR/Passes/ImageProcessPass.hpp b/src/libRender/RadiumNBR/Passes/ImageProcessPass.hpp
new file mode 100644
index 0000000..169ddbe
--- /dev/null
+++ b/src/libRender/RadiumNBR/Passes/ImageProcessPass.hpp
@@ -0,0 +1,59 @@
+#pragma once
+#include <RadiumNBR/RenderPass.hpp>
+
+#include <Engine/Data/DisplayableObject.hpp>
+
+namespace Ra::Engine::Data {
+class ShaderProgram;
+} // namespace Ra::Engine::Data
+
+namespace globjects {
+class Framebuffer;
+}
+
+namespace RadiumNBR {
+/// Render pass do 2D processing on input textures to generate a resulting color texture.
+class ImageProcessPass : public RenderPass
+{
+ public:
+ explicit ImageProcessPass( const Ra::Core::Utils::Index& idx );
+ ~ImageProcessPass() override;
+
+ bool buildRenderTechnique( const Ra::Engine::Rendering::RenderObject* ro,
+ Ra::Engine::Rendering::RenderTechnique& rt ) const override;
+ bool initializePass( size_t width,
+ size_t height,
+ Ra::Engine::Data::ShaderProgramManager* shaderMngr ) override;
+ bool update() override;
+ void resize( size_t width, size_t height ) override;
+ void execute( const Ra::Engine::Data::ViewingParameters& viewParams ) const override;
+
+ /// Add the output colorBuffer.
+ /// \warning the output color buffer must be different from the input one
+ /// \todo allow to have same input/output color buffer
+ void setOutput( const SharedTextures& colorBuffer );
+
+ /// These inputs must be computed before executing this pass : depth buffer
+ void setInputs( const SharedTextures& depthBuffer, const SharedTextures& colorBuffer );
+
+ void rebuildShaders();
+
+ private:
+ /// The framebuffer used to render this pass
+ std::unique_ptr<globjects::Framebuffer> m_fbo{nullptr};
+
+ /// The Shader manager to use when building shaders
+ Ra::Engine::Data::ShaderProgramManager* m_shaderMngr{nullptr};
+
+ /// The color texture for output.Stored here for easy access.
+ SharedTextures m_outputTexture;
+
+ /// The quad to be drawn for shader invocation
+ std::unique_ptr<Ra::Engine::Data::Displayable> m_quadMesh{nullptr};
+
+ /// The shader that process the image (the Radium shader manager has ownership)
+ const Ra::Engine::Data::ShaderProgram* m_shader{nullptr};
+
+
+};
+} // namespace RadiumNBR
diff --git a/src/libRender/RadiumNBR/Renderer/Visualization.cpp b/src/libRender/RadiumNBR/Renderer/Visualization.cpp
index c415fc7..333d32d 100644
--- a/src/libRender/RadiumNBR/Renderer/Visualization.cpp
+++ b/src/libRender/RadiumNBR/Renderer/Visualization.cpp
@@ -3,6 +3,7 @@
#include <RadiumNBR/Passes/ClearPass.hpp>
#include <RadiumNBR/Passes/CustomAttribToColorPass.hpp>
#include <RadiumNBR/Passes/GeomPrepass.hpp>
+#include <RadiumNBR/Passes/ImageProcessPass.hpp>
#include <RadiumNBR/Renderer/Visualization.hpp>
#include <Core/Utils/Log.hpp>
@@ -36,7 +37,7 @@ void VisualizationController::configure( RadiumNBR::NodeBasedRenderer* renderer,
// set the output of the pass : clear the renderer Linear RGB output texture
m_clearPass->setOutput( *colortexture );
// set the clear color (pass internal state)
- m_clearPass->setBackground( {0.5, 0.1, 0.1} /*renderer->getBackgroundColor()*/ );
+ m_clearPass->setBackground( renderer->getBackgroundColor() );
m_clearPass->initializePass( w, h, shaderManager );
// add the pass to the renderer and activate it
renderer->addPass( m_clearPass, m_clearPass->index() );
@@ -70,6 +71,28 @@ void VisualizationController::configure( RadiumNBR::NodeBasedRenderer* renderer,
m_customPass->activate();
}
//! [Adding a CustomAttribToColorPass pass]
+
+ //! [Adding an image processing pass]
+ {
+ // pass that draw no object and is positioned at rank 0
+ m_imageProcessPass = std::make_shared<RadiumNBR::ImageProcessPass>(3 );
+ // set the output of the pass : clear the renderer Linear RGB output texture
+ auto vectorFieldTex = renderer->sharedTextures().find( "CustomAtt2Clr::VectorField" );
+ m_imageProcessPass->setInputs( *depthtexture, *vectorFieldTex );
+ m_imageProcessPass->setOutput( *colortexture );
+ // configure access to shader/resources files and initialize the pass
+ m_imageProcessPass->setResourcesDir( resourcesPath );
+ m_imageProcessPass->initializePass( w, h, shaderManager );
+ // add the pass to the renderer and activate it
+ renderer->addPass( m_imageProcessPass, m_imageProcessPass->index() );
+ m_imageProcessPass->activate();
+ // Add the exported (shared) texture to the set of available textures
+ auto &sharedTextures = renderer->sharedTextures();
+ for (auto t : m_imageProcessPass->getOutputImages() ) {
+ sharedTextures.insert( {t.first, t.second} );
+ }
+ }
+ //! [Adding a clear-screen pass]
}
void VisualizationController::update( const Ra::Engine::Data::ViewingParameters& ) {
diff --git a/src/libRender/RadiumNBR/Renderer/Visualization.hpp b/src/libRender/RadiumNBR/Renderer/Visualization.hpp
index 9f18f05..8296e46 100644
--- a/src/libRender/RadiumNBR/Renderer/Visualization.hpp
+++ b/src/libRender/RadiumNBR/Renderer/Visualization.hpp
@@ -5,6 +5,7 @@
namespace RadiumNBR {
class CustomAttribToColorPass;
class ClearPass;
+class ImageProcessPass;
/**
* This class parameterize the renderer just after the OpenGL system was initialized.
@@ -53,6 +54,11 @@ class NodeBasedRenderer_LIBRARY_API VisualizationController
/// The clear pass for the background
std::shared_ptr<RadiumNBR::ClearPass> m_clearPass;
+
+ /// The image processing pass to render vector field
+ std::shared_ptr<RadiumNBR::ImageProcessPass> m_imageProcessPass;
+
+
/// TODO : this default are the same as in the CustomAttribToColorPass. Use them instead of redefining here
std::string m_vertexFunction{"void outputCustomAttribs() {\n}\n"};
std::string m_geometryFunction{"void propagateAttributes(){}\n"};
diff --git a/src/libRender/RadiumNBR/Shaders/ImageProcessingPass/ImageProcess.frag.glsl b/src/libRender/RadiumNBR/Shaders/ImageProcessingPass/ImageProcess.frag.glsl
new file mode 100644
index 0000000..721692b
--- /dev/null
+++ b/src/libRender/RadiumNBR/Shaders/ImageProcessingPass/ImageProcess.frag.glsl
@@ -0,0 +1,77 @@
+layout (location = 0) out vec4 out_color;
+
+uniform sampler2D depth_sampler;
+uniform sampler2D image_sampler;
+in vec2 varTexcoord;
+
+
+// replace the nois texture : TODO verify the properties of the generated noise
+float whiteNoise(in vec2 at, in vec2 scale) {
+ return (noise1 (at * scale) + 0.5) * 0.5;
+}
+
+/*
+// LIC function to be adapted ...
+//desc : texture de direction (vecteurs)
+// invertdir pour faire "perpendiculaire" a desc,
+// noise une texture de bruit "poivre et sel"
+// sw, sh : screensize
+vec4 lic(in sampler2D desc,in bool invertdir) {
+ const int halfsize = 10;
+ vec2 coord;
+ vec2 dir = texture2D(desc,gl_TexCoord[0].st).xy;
+ //vec2 dir = texelFetch(desc,ivec2(gl_TexCoord[0].st*1.0/vec2(sw,sh)),0).xy;
+ if(invertdir)
+ dir = vec2(dir.y,-dir.x);
+ vec4 res = texture2D(noise,gl_TexCoord[0].st);
+ vec2 currentdir;
+ vec2 tmpdir;
+ coord = gl_TexCoord[0].st;
+ currentdir = dir;
+ for(int i=1;i<=halfsize;i++) {
+ coord = coord+vec2(currentdir.x*sw,currentdir.y*sh);
+ res += texture2D(noise,coord);
+ tmpdir = texture2D(desc,coord).xy;
+ //tmpdir = texelFetch(desc,ivec2(coord*1.0/vec2(sw,sh)),0).xy;
+ if(invertdir)
+ tmpdir = vec2(tmpdir.y,-tmpdir.x);
+ currentdir = tmpdir;
+ }
+ coord = gl_TexCoord[0].st;
+ currentdir = dir;
+ for(int i=1;i<=halfsize;i++) {
+ coord = coord-vec2(currentdir.x*sw,currentdir.y*sh);
+ res += texture2D(noise,coord);
+ tmpdir = texture2D(desc,coord).xy;
+ //tmpdir = texelFetch(desc,ivec2(coord*1.0/vec2(sw,sh)),0).xy;
+ if(invertdir)
+ tmpdir = vec2(tmpdir.y,-tmpdir.x);
+ currentdir = tmpdir;
+ }
+ res = res/(2.0f*float(halfsize)+1.0f);
+ return vec4(res.x);
+}
+
+*/
+
+void main() {
+ /*
+ // 4x4 filtering for test
+ const int half_width = 2;
+ vec2 texelSize = 1.0 / vec2(textureSize(image_sampler, 0));
+ vec3 result = vec3(0.0);
+ for (int x = -half_width; x < half_width; ++x)
+ {
+ for (int y = -half_width; y < half_width; ++y)
+ {
+ vec2 offset = vec2(float(x), float(y)) * texelSize;
+ result += texture(image_sampler, varTexcoord + offset).rgb;
+ }
+ }
+ out_color = vec4(result / (4 * half_width * half_width), 1);
+ */
+
+ vec2 imgSize = vec2(textureSize(image_sampler, 0));
+ out_color = vec4( vec3( whiteNoise( varTexcoord, imgSize) ), 1);
+}
+
diff --git a/src/libRender/RadiumNBR/Shaders/ImageProcessingPass/ImageProcess.vert.glsl b/src/libRender/RadiumNBR/Shaders/ImageProcessingPass/ImageProcess.vert.glsl
new file mode 100644
index 0000000..c540dd1
--- /dev/null
+++ b/src/libRender/RadiumNBR/Shaders/ImageProcessingPass/ImageProcess.vert.glsl
@@ -0,0 +1,7 @@
+layout (location = 0) in vec3 in_position;
+out vec2 varTexcoord;
+void main(void)
+{
+ gl_Position = vec4(in_position.xyz, 1.0);
+ varTexcoord = (in_position.xy + 1.0) / 2.0;
+}
--
GitLab