Fix additional overflow and underflow issues with triangle fans and line loops.
Issue #444
Signed-off-by: Shannon Woods
Signed-off-by: Chris Evans
Author: Geoff Lang
Conflicts:
src/common/version.h
src/libGLESv2/renderer/Renderer11.cpp
diff --git a/src/common/version.h b/src/common/version.h
index bb1963d..8ad6e2f 100644
--- a/src/common/version.h
+++ b/src/common/version.h
@@ -1,7 +1,7 @@
#define MAJOR_VERSION 1
#define MINOR_VERSION 29
#define BUILD_VERSION 0
-#define BUILD_REVISION 2423
+#define BUILD_REVISION 2424
#define STRINGIFY(x) #x
#define MACRO_STRINGIFY(x) STRINGIFY(x)
diff --git a/src/libGLESv2/renderer/Renderer11.cpp b/src/libGLESv2/renderer/Renderer11.cpp
index 5243c95..7af2b2f 100644
--- a/src/libGLESv2/renderer/Renderer11.cpp
+++ b/src/libGLESv2/renderer/Renderer11.cpp
@@ -835,23 +835,25 @@
{
D3D11_PRIMITIVE_TOPOLOGY primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_UNDEFINED;
+ GLsizei minCount = 0;
+
switch (mode)
{
- case GL_POINTS: primitiveTopology = D3D11_PRIMITIVE_TOPOLOGY_POINTLIST; break;
- case GL_LINES: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINELIST; break;
- case GL_LINE_LOOP: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINESTRIP; break;
- case GL_LINE_STRIP: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINESTRIP; break;
- case GL_TRIANGLES: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST; break;
- case GL_TRIANGLE_STRIP: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP; break;
+ case GL_POINTS: primitiveTopology = D3D11_PRIMITIVE_TOPOLOGY_POINTLIST; minCount = 1; break;
+ case GL_LINES: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINELIST; minCount = 2; break;
+ case GL_LINE_LOOP: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINESTRIP; minCount = 2; break;
+ case GL_LINE_STRIP: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_LINESTRIP; minCount = 2; break;
+ case GL_TRIANGLES: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST; minCount = 3; break;
+ case GL_TRIANGLE_STRIP: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP; minCount = 3; break;
// emulate fans via rewriting index buffer
- case GL_TRIANGLE_FAN: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST; break;
+ case GL_TRIANGLE_FAN: primitiveTopology = D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST; minCount = 3; break;
default:
return gl::error(GL_INVALID_ENUM, false);
}
mDeviceContext->IASetPrimitiveTopology(primitiveTopology);
- return count > 0;
+ return count >= minCount;
}
bool Renderer11::applyRenderTarget(gl::Framebuffer *framebuffer)
@@ -1130,13 +1132,16 @@
}
}
- if (static_cast<unsigned int>(count + 1) > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
+ // Checked by Renderer11::applyPrimitiveType
+ ASSERT(count >= 0);
+
+ if (static_cast<unsigned int>(count) + 1 > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
{
ERR("Could not create a 32-bit looping index buffer for GL_LINE_LOOP, too many indices required.");
return gl::error(GL_OUT_OF_MEMORY);
}
- const unsigned int spaceNeeded = (count + 1) * sizeof(unsigned int);
+ const unsigned int spaceNeeded = (static_cast<unsigned int>(count) + 1) * sizeof(unsigned int);
if (!mLineLoopIB->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_INT))
{
ERR("Could not reserve enough space in looping index buffer for GL_LINE_LOOP.");
@@ -1230,9 +1235,12 @@
}
}
+ // Checked by Renderer11::applyPrimitiveType
+ ASSERT(count >= 3);
+
const unsigned int numTris = count - 2;
- if (numTris * 3 > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
+ if (numTris > (std::numeric_limits<unsigned int>::max() / (sizeof(unsigned int) * 3)))
{
ERR("Could not create a scratch index buffer for GL_TRIANGLE_FAN, too many indices required.");
return gl::error(GL_OUT_OF_MEMORY);
diff --git a/src/libGLESv2/renderer/Renderer9.cpp b/src/libGLESv2/renderer/Renderer9.cpp
index 4aecd4a..240367d 100644
--- a/src/libGLESv2/renderer/Renderer9.cpp
+++ b/src/libGLESv2/renderer/Renderer9.cpp
@@ -1471,13 +1471,16 @@
}
}
- if (static_cast<unsigned int>(count + 1) > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
+ if (static_cast<unsigned int>(count) + 1 > (std::numeric_limits<unsigned int>::max() / sizeof(unsigned int)))
{
ERR("Could not create a 32-bit looping index buffer for GL_LINE_LOOP, too many indices required.");
return gl::error(GL_OUT_OF_MEMORY);
}
- const unsigned int spaceNeeded = (count + 1) * sizeof(unsigned int);
+ // Checked by Renderer9::applyPrimitiveType
+ ASSERT(count >= 0);
+
+ const unsigned int spaceNeeded = (static_cast<unsigned int>(count) + 1) * sizeof(unsigned int);
if (!mLineLoopIB->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_INT))
{
ERR("Could not reserve enough space in looping index buffer for GL_LINE_LOOP.");
@@ -1549,13 +1552,16 @@
}
}
- if (static_cast<unsigned int>(count + 1) > (std::numeric_limits<unsigned short>::max() / sizeof(unsigned short)))
+ // Checked by Renderer9::applyPrimitiveType
+ ASSERT(count >= 0);
+
+ if (static_cast<unsigned int>(count) + 1 > (std::numeric_limits<unsigned short>::max() / sizeof(unsigned short)))
{
ERR("Could not create a 16-bit looping index buffer for GL_LINE_LOOP, too many indices required.");
return gl::error(GL_OUT_OF_MEMORY);
}
- const int spaceNeeded = (count + 1) * sizeof(unsigned short);
+ const unsigned int spaceNeeded = (static_cast<unsigned int>(count) + 1) * sizeof(unsigned short);
if (!mLineLoopIB->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_SHORT))
{
ERR("Could not reserve enough space in looping index buffer for GL_LINE_LOOP.");
