Correct clamped depth range from [0, 65535].

This changes a few things:
 * All backends clamp the depth range and keep it positive.
 * The depth rounding uniform is now properly dirtied.
 * Projection is updated to translate and scale appropriately.
 * Depth rounding is halved on OpenGL to account for [-1, 1] range.

Fixes Phantasy Star Portable 2 without the need for a game-specific hack.

Author: unknownbrackets

GPU/Common/GPUStateUtils.cpp

@@ -597,6 +597,8 @@ void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, flo
 		float xOffset = 0.0f;
 		float hScale = 1.0f;
 		float yOffset = 0.0f;
+		float zScale = 1.0f;
+		float zOffset = 0.0f;
 
 		// If we're within the bounds, we want clipping the viewport way.  So leave it be.
 		if (left < 0.0f || right > renderWidth) {
@@ -625,28 +627,54 @@ void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, flo
 			yOffset = drift / (bottom - top);
 		}
 
+		out.viewportX = left + displayOffsetX;
+		out.viewportY = top + displayOffsetY;
+		out.viewportW = right - left;
+		out.viewportH = bottom - top;
+
+		float vpZScale = gstate.getViewportZScale();
+		float vpZCenter = gstate.getViewportZCenter();
+		float depthRangeMin = vpZCenter - vpZScale;
+		float depthRangeMax = vpZCenter + vpZScale;
+		// Near/far can be inverted.  Let's reverse while dealing with clamping, though.
+		bool inverted = vpZScale < 0.0f;
+		float near = (inverted ? depthRangeMax : depthRangeMin) * (1.0f / 65535.0f);
+		float far = (inverted ? depthRangeMin : depthRangeMax) * (1.0f / 65535.0f);
+
+		if (near < 0.0f || far > 1.0f) {
+			float overageNear = std::max(-near, 0.0f);
+			float overageFar = std::max(far - 1.0f, 0.0f);
+			float drift = overageFar - overageNear;
+
+			near += overageNear;
+			far += overageFar;
+
+			zScale = fabsf(vpZScale * (2.0f / 65535.0f)) / (far - near);
+			zOffset = drift / (far - near);
+		}
+
+		if (inverted) {
+			zScale = -zScale;
+			inverted = false;
+		}
+
+		out.depthRangeMin = inverted ? far : near;
+		out.depthRangeMax = inverted ? near : far;
+
 		bool scaleChanged = gstate_c.vpWidthScale != wScale || gstate_c.vpHeightScale != hScale;
 		bool offsetChanged = gstate_c.vpXOffset != xOffset || gstate_c.vpYOffset != yOffset;
-		if (scaleChanged || offsetChanged) {
+		bool depthChanged = gstate_c.vpDepthScale != zScale || gstate_c.vpZOffset != zOffset;
+		if (scaleChanged || offsetChanged || depthChanged) {
 			gstate_c.vpWidthScale = wScale;
 			gstate_c.vpHeightScale = hScale;
+			gstate_c.vpDepthScale = zScale;
 			gstate_c.vpXOffset = xOffset;
 			gstate_c.vpYOffset = yOffset;
+			gstate_c.vpZOffset = zOffset;
 			out.dirtyProj = true;
+			out.dirtyDepth = depthChanged;
 		}
 
-		out.viewportX = left + displayOffsetX;
-		out.viewportY = top + displayOffsetY;
-		out.viewportW = right - left;
-		out.viewportH = bottom - top;
-
-		float zScale = gstate.getViewportZScale();
-		float zCenter = gstate.getViewportZCenter();
-		float depthRangeMin = zCenter - zScale;
-		float depthRangeMax = zCenter + zScale;
-		out.depthRangeMin = depthRangeMin * (1.0f / 65535.0f);
-		out.depthRangeMax = depthRangeMax * (1.0f / 65535.0f);
-
 #ifndef MOBILE_DEVICE
 		float minz = gstate.getDepthRangeMin();
 		float maxz = gstate.getDepthRangeMax();

GPU/Common/GPUStateUtils.h

@@ -63,6 +63,7 @@ struct ViewportAndScissor {
 	float depthRangeMin;
 	float depthRangeMax;
 	bool dirtyProj;
+	bool dirtyDepth;
 };
 void ConvertViewportAndScissor(bool useBufferedRendering, float renderWidth, float renderHeight, int bufferWidth, int bufferHeight, ViewportAndScissor &out);
 

GPU/Directx9/ShaderManagerDX9.cpp

@@ -248,7 +248,7 @@ void ShaderManagerDX9::VSSetMatrix(int creg, const float* pMatrix) {
 }
 
 // Depth in ogl is between -1;1 we need between 0;1 and optionally reverse it
-static void ConvertProjMatrixToD3D(Matrix4x4 &in, bool invertedX, bool invertedY, bool invertedZ) {
+static void ConvertProjMatrixToD3D(Matrix4x4 &in, bool invertedX, bool invertedY) {
 	// Half pixel offset hack
 	float xoff = 0.5f / gstate_c.curRTRenderWidth;
 	xoff = gstate_c.vpXOffset + (invertedX ? xoff : -xoff);
@@ -260,7 +260,9 @@ static void ConvertProjMatrixToD3D(Matrix4x4 &in, bool invertedX, bool invertedY
 	if (invertedY)
 		yoff = -yoff;
 
-	in.translateAndScale(Vec3(xoff, yoff, 0.5f), Vec3(gstate_c.vpWidthScale, gstate_c.vpHeightScale, invertedZ ? -0.5 : 0.5f));
+	const Vec3 trans(xoff, yoff, gstate_c.vpZOffset * 0.5f + 0.5f);
+	const Vec3 scale(gstate_c.vpWidthScale, gstate_c.vpHeightScale, gstate_c.vpDepthScale * 0.5f);
+	in.translateAndScale(trans, scale);
 }
 
 static void ConvertProjMatrixToD3DThrough(Matrix4x4 &in) {
@@ -342,8 +344,7 @@ void ShaderManagerDX9::VSUpdateUniforms(int dirtyUniforms) {
 			flippedMatrix[12] = -flippedMatrix[12];
 		}
 
-		const bool invertedZ = gstate_c.vpDepth < 0;
-		ConvertProjMatrixToD3D(flippedMatrix, invertedX, invertedY, invertedZ);
+		ConvertProjMatrixToD3D(flippedMatrix, invertedX, invertedY);
 
 		VSSetMatrix(CONST_VS_PROJ, flippedMatrix.getReadPtr());
 	}
@@ -482,16 +483,21 @@ void ShaderManagerDX9::VSUpdateUniforms(int dirtyUniforms) {
 		float viewZScale = gstate.getViewportZScale();
 		float viewZCenter = gstate.getViewportZCenter();
 
-		// Given the way we do the rounding, the integer part of the offset is probably mostly irrelevant as we cancel
-		// it afterwards anyway.
-		// It seems that we should adjust for D3D projection matrix. We got squashed up to only 0-1, so we divide
-		// the scale factor by 2, and add an offset. But, this doesn't work! I get near-perfect results not doing it.
-		// viewZScale *= 2.0f;
+		// We had to scale and translate Z to account for our clamped Z range.
+		// Therefore, we also need to reverse this to round properly.
+		//
+		// Example: scale = 65535.0, center = 0.0
+		// Resulting range = -65535 to 65535, clamped to [0, 65535]
+		// gstate_c.vpDepthScale = 2.0f
+		// gstate_c.vpZOffset = -1.0f
+		//
+		// The projection already accounts for those, so we need to reverse them.
+		//
+		// Additionally, D3D9 uses a range from [0, 1] which makes things easy.
+		// We just correct the center.
+		viewZScale *= (1.0f / gstate_c.vpDepthScale);
+		viewZCenter -= 65535.0f * gstate_c.vpZOffset - 32767.5f;
 
-		// Need to take the possibly inverted proj matrix into account.
-		if (gstate_c.vpDepth < 0.0)
-			viewZScale *= -1.0f;
-		viewZCenter -= 32767.5f;
 		float viewZInvScale;
 		if (viewZScale != 0.0) {
 			viewZInvScale = 1.0f / viewZScale;

GPU/Directx9/StateMappingDX9.cpp

@@ -293,23 +293,13 @@ void TransformDrawEngineDX9::ApplyDrawState(int prim) {
 	float depthMin = vpAndScissor.depthRangeMin;
 	float depthMax = vpAndScissor.depthRangeMax;
 
-	if (!gstate.isModeThrough()) {
-		// Direct3D can't handle negative depth ranges, so we fix it in the projection matrix.
-		if (gstate_c.vpDepth != depthMax - depthMin) {
-			gstate_c.vpDepth = depthMax - depthMin;
-			vpAndScissor.dirtyProj = true;
-		}
-		if (depthMin > depthMax) {
-			std::swap(depthMin, depthMax);
-		}
-		if (depthMin < 0.0f) depthMin = 0.0f;
-		if (depthMax > 1.0f) depthMax = 1.0f;
-	}
-
 	dxstate.viewport.set(vpAndScissor.viewportX, vpAndScissor.viewportY, vpAndScissor.viewportW, vpAndScissor.viewportH, depthMin, depthMax);
 	if (vpAndScissor.dirtyProj) {
 		shaderManager_->DirtyUniform(DIRTY_PROJMATRIX);
 	}
+	if (vpAndScissor.dirtyDepth) {
+		shaderManager_->DirtyUniform(DIRTY_DEPTHRANGE);
+	}
 }
 
 void TransformDrawEngineDX9::ApplyDrawStateLate() {

GPU/GLES/ShaderManager.cpp

@@ -378,8 +378,8 @@ static inline void ScaleProjMatrix(Matrix4x4 &in) {
 		// GL upside down is a pain as usual.
 		yOffset = -yOffset;
 	}
-	const Vec3 trans(gstate_c.vpXOffset, yOffset, 0.0f);
-	const Vec3 scale(gstate_c.vpWidthScale, gstate_c.vpHeightScale, 1.0);
+	const Vec3 trans(gstate_c.vpXOffset, yOffset, gstate_c.vpZOffset * 2.0f);
+	const Vec3 scale(gstate_c.vpWidthScale, gstate_c.vpHeightScale, gstate_c.vpDepthScale);
 	in.translateAndScale(trans, scale);
 }
 
@@ -592,11 +592,27 @@ void LinkedShader::UpdateUniforms(u32 vertType) {
 		float viewZScale = gstate.getViewportZScale();
 		float viewZCenter = gstate.getViewportZCenter();
 		float viewZInvScale;
+
+		// We had to scale and translate Z to account for our clamped Z range.
+		// Therefore, we also need to reverse this to round properly.
+		//
+		// Example: scale = 65535.0, center = 0.0
+		// Resulting range = -65535 to 65535, clamped to [0, 65535]
+		// gstate_c.vpDepthScale = 2.0f
+		// gstate_c.vpZOffset = -1.0f
+		//
+		// The projection already accounts for those, so we need to reverse them.
+		//
+		// Additionally, OpenGL uses a range from [-1, 1].  So we halve the scale.
+		viewZScale *= (1.0f / gstate_c.vpDepthScale) * 0.5f;
+		viewZCenter -= 65535.0f * gstate_c.vpZOffset;
+
 		if (viewZScale != 0.0) {
 			viewZInvScale = 1.0f / viewZScale;
 		} else {
 			viewZInvScale = 0.0;
 		}
+
 		float data[4] = { viewZScale, viewZCenter, viewZCenter, viewZInvScale };
 		SetFloatUniform4(u_depthRange, data);
 	}

GPU/GLES/StateMapping.cpp

@@ -371,6 +371,9 @@ void TransformDrawEngine::ApplyDrawState(int prim) {
 	if (vpAndScissor.dirtyProj) {
 		shaderManager_->DirtyUniform(DIRTY_PROJMATRIX);
 	}
+	if (vpAndScissor.dirtyDepth) {
+		shaderManager_->DirtyUniform(DIRTY_DEPTHRANGE);
+	}
 }
 
 void TransformDrawEngine::ApplyDrawStateLate() {

GPU/GPUState.cpp

@@ -254,10 +254,9 @@ void GPUStateCache::DoState(PointerWrap &p) {
 
 	p.Do(vpWidth);
 	p.Do(vpHeight);
-	if (s >= 4) {
-		p.Do(vpDepth);
-	} else {
-		vpDepth = 1.0f;  // any positive value should be fine
+	if (s == 4) {
+		float oldDepth = 1.0f;
+		p.Do(oldDepth);
 	}
 
 	p.Do(curRTWidth);

GPU/GPUState.h

@@ -510,12 +510,13 @@ struct GPUStateCache {
 
 	float vpWidth;
 	float vpHeight;
-	float vpDepth;
 
 	float vpXOffset;
 	float vpYOffset;
+	float vpZOffset;
 	float vpWidthScale;
 	float vpHeightScale;
+	float vpDepthScale;
 
 	KnownVertexBounds vertBounds;