Skip to content

Commit e9258bd

Browse files
BradLarsonBradLarson
authored
Merge pull request BradLarson#91 from joshbernfeld/buffer-size
Fix incorrect buffer size and offsets by using MTLAutoreleasedRenderPipelineReflection
2 parents 0e188be + 9fb0e72 commit e9258bd

File tree

9 files changed

+38
-68
lines changed

9 files changed

+38
-68
lines changed

‎framework/Source/BasicOperation.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,9 @@ open class BasicOperation: ImageProcessingOperation {
4141
self.operationName = operationName
4242

4343
let concreteVertexFunctionName = vertexFunctionName ?? defaultVertexFunctionNameForInputs(numberOfInputs)
44-
let (pipelineState, lookupTable) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:concreteVertexFunctionName, fragmentFunctionName:fragmentFunctionName, operationName:operationName)
44+
let (pipelineState, lookupTable, bufferSize) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:concreteVertexFunctionName, fragmentFunctionName:fragmentFunctionName, operationName:operationName)
4545
self.renderPipelineState = pipelineState
46-
self.uniformSettings = ShaderUniformSettings(uniformLookupTable:lookupTable)
46+
self.uniformSettings = ShaderUniformSettings(uniformLookupTable:lookupTable, bufferSize:bufferSize)
4747
}
4848

4949
public func transmitPreviousImage(to target: ImageConsumer, atIndex: UInt) {

‎framework/Source/Camera.swift‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,8 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer
6868
var supportsFullYUVRange:Bool = false
6969
let captureAsYUV:Bool
7070
let yuvConversionRenderPipelineState:MTLRenderPipelineState?
71-
var yuvLookupTable:[String:(Int, MTLDataType)] = [:]
71+
var yuvLookupTable:[String:(Int, MTLStructMember)] = [:]
72+
var yuvBufferSize:Int = 0
7273

7374
let frameRenderingSemaphore = DispatchSemaphore(value:1)
7475
let cameraProcessingQueue = DispatchQueue.global()
@@ -133,15 +134,17 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer
133134
}
134135
}
135136
if (supportsFullYUVRange) {
136-
let (pipelineState, lookupTable) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"twoInputVertex", fragmentFunctionName:"yuvConversionFullRangeFragment", operationName:"YUVToRGB")
137+
let (pipelineState, lookupTable, bufferSize) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"twoInputVertex", fragmentFunctionName:"yuvConversionFullRangeFragment", operationName:"YUVToRGB")
137138
self.yuvConversionRenderPipelineState = pipelineState
138139
self.yuvLookupTable = lookupTable
140+
self.yuvBufferSize = bufferSize
139141
videoOutput.videoSettings = [kCVPixelBufferMetalCompatibilityKey as String: true,
140142
kCVPixelBufferPixelFormatTypeKey as String:NSNumber(value:Int32(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange))]
141143
} else {
142-
let (pipelineState, lookupTable) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"twoInputVertex", fragmentFunctionName:"yuvConversionVideoRangeFragment", operationName:"YUVToRGB")
144+
let (pipelineState, lookupTable, bufferSize) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"twoInputVertex", fragmentFunctionName:"yuvConversionVideoRangeFragment", operationName:"YUVToRGB")
143145
self.yuvConversionRenderPipelineState = pipelineState
144146
self.yuvLookupTable = lookupTable
147+
self.yuvBufferSize = bufferSize
145148
videoOutput.videoSettings = [kCVPixelBufferMetalCompatibilityKey as String: true,
146149
kCVPixelBufferPixelFormatTypeKey as String:NSNumber(value:Int32(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange))]
147150
}
@@ -219,7 +222,7 @@ public class Camera: NSObject, ImageSource, AVCaptureVideoDataOutputSampleBuffer
219222
}
220223
let outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:.portrait, width:outputWidth, height:outputHeight, timingStyle: .videoFrame(timestamp: Timestamp(currentTime)))
221224

222-
convertYUVToRGB(pipelineState:self.yuvConversionRenderPipelineState!, lookupTable:self.yuvLookupTable,
225+
convertYUVToRGB(pipelineState:self.yuvConversionRenderPipelineState!, lookupTable:self.yuvLookupTable, bufferSize:self.yuvBufferSize,
223226
luminanceTexture:Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture:luminanceTexture),
224227
chrominanceTexture:Texture(orientation: self.orientation ?? self.location.imageOrientation(), texture:chrominanceTexture),
225228
resultTexture:outputTexture, colorConversionMatrix:conversionMatrix)

‎framework/Source/MetalRendering.swift‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ extension MTLCommandBuffer {
6262
}
6363
}
6464

65-
func generateRenderPipelineState(device:MetalRenderingDevice, vertexFunctionName:String, fragmentFunctionName:String, operationName:String) -> (MTLRenderPipelineState, [String:(Int, MTLDataType)]) {
65+
func generateRenderPipelineState(device:MetalRenderingDevice, vertexFunctionName:String, fragmentFunctionName:String, operationName:String) -> (MTLRenderPipelineState, [String:(Int, MTLStructMember)], Int) {
6666
guard let vertexFunction = device.shaderLibrary.makeFunction(name: vertexFunctionName) else {
6767
fatalError("\(operationName): could not compile vertex function \(vertexFunctionName)")
6868
}
@@ -81,20 +81,22 @@ func generateRenderPipelineState(device:MetalRenderingDevice, vertexFunctionName
8181
var reflection:MTLAutoreleasedRenderPipelineReflection?
8282
let pipelineState = try device.device.makeRenderPipelineState(descriptor: descriptor, options: [.bufferTypeInfo, .argumentInfo], reflection: &reflection)
8383

84-
var uniformLookupTable:[String:(Int, MTLDataType)] = [:]
84+
var uniformLookupTable:[String:(Int, MTLStructMember)] = [:]
85+
var bufferSize: Int = 0
8586
if let fragmentArguments = reflection?.fragmentArguments {
8687
for fragmentArgument in fragmentArguments where fragmentArgument.type == .buffer {
8788
if
8889
(fragmentArgument.bufferDataType == .struct),
8990
let members = fragmentArgument.bufferStructType?.members.enumerated() {
91+
bufferSize = fragmentArgument.bufferDataSize
9092
for (index, uniform) in members {
91-
uniformLookupTable[uniform.name] = (index, uniform.dataType)
93+
uniformLookupTable[uniform.name] = (index, uniform)
9294
}
9395
}
9496
}
9597
}
9698

97-
return (pipelineState, uniformLookupTable)
99+
return (pipelineState, uniformLookupTable, bufferSize)
98100
} catch {
99101
fatalError("Could not create render pipeline state for vertex:\(vertexFunctionName), fragment:\(fragmentFunctionName), error:\(error)")
100102
}

‎framework/Source/MetalRenderingDevice.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ public class MetalRenderingDevice {
1414
public let metalPerformanceShadersAreSupported: Bool
1515

1616
lazy var passthroughRenderState: MTLRenderPipelineState = {
17-
let (pipelineState, _) = generateRenderPipelineState(device:self, vertexFunctionName:"oneInputVertex", fragmentFunctionName:"passthroughFragment", operationName:"Passthrough")
17+
let (pipelineState, _, _) = generateRenderPipelineState(device:self, vertexFunctionName:"oneInputVertex", fragmentFunctionName:"passthroughFragment", operationName:"Passthrough")
1818
return pipelineState
1919
}()
2020

2121
lazy var colorSwizzleRenderState: MTLRenderPipelineState = {
22-
let (pipelineState, _) = generateRenderPipelineState(device:self, vertexFunctionName:"oneInputVertex", fragmentFunctionName:"colorSwizzleFragment", operationName:"ColorSwizzle")
22+
let (pipelineState, _, _) = generateRenderPipelineState(device:self, vertexFunctionName:"oneInputVertex", fragmentFunctionName:"colorSwizzleFragment", operationName:"ColorSwizzle")
2323
return pipelineState
2424
}()
2525

‎framework/Source/MovieInput.swift‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@ public class MovieInput: ImageSource {
77

88
var videoTextureCache: CVMetalTextureCache?
99
let yuvConversionRenderPipelineState:MTLRenderPipelineState
10-
var yuvLookupTable:[String:(Int, MTLDataType)] = [:]
11-
10+
var yuvLookupTable:[String:(Int, MTLStructMember)] = [:]
11+
var yuvBufferSize:Int = 0
12+
1213
let asset:AVAsset
1314
let assetReader:AVAssetReader
1415
let playAtActualSpeed:Bool
@@ -26,9 +27,10 @@ public class MovieInput: ImageSource {
2627
self.asset = asset
2728
self.playAtActualSpeed = playAtActualSpeed
2829
self.loop = loop
29-
let (pipelineState, lookupTable) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"twoInputVertex", fragmentFunctionName:"yuvConversionFullRangeFragment", operationName:"YUVToRGB")
30+
let (pipelineState, lookupTable, bufferSize) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"twoInputVertex", fragmentFunctionName:"yuvConversionFullRangeFragment", operationName:"YUVToRGB")
3031
self.yuvConversionRenderPipelineState = pipelineState
3132
self.yuvLookupTable = lookupTable
33+
self.yuvBufferSize = bufferSize
3234
let _ = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, sharedMetalRenderingDevice.device, nil, &videoTextureCache)
3335

3436
assetReader = try AVAssetReader(asset:self.asset)
@@ -177,7 +179,7 @@ public class MovieInput: ImageSource {
177179
let luminanceTexture = CVMetalTextureGetTexture(concreteLuminanceTextureRef), let chrominanceTexture = CVMetalTextureGetTexture(concreteChrominanceTextureRef) {
178180
let outputTexture = Texture(device:sharedMetalRenderingDevice.device, orientation:.portrait, width:bufferWidth, height:bufferHeight, timingStyle:.videoFrame(timestamp:Timestamp(withSampleTime)))
179181

180-
convertYUVToRGB(pipelineState:self.yuvConversionRenderPipelineState, lookupTable:self.yuvLookupTable,
182+
convertYUVToRGB(pipelineState:self.yuvConversionRenderPipelineState, lookupTable:self.yuvLookupTable, bufferSize:self.yuvBufferSize,
181183
luminanceTexture:Texture(orientation:.portrait, texture:luminanceTexture),
182184
chrominanceTexture:Texture(orientation:.portrait, texture:chrominanceTexture),
183185
resultTexture:outputTexture, colorConversionMatrix:conversionMatrix)

‎framework/Source/MovieOutput.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ public class MovieOutput: ImageConsumer, AudioEncodingTarget {
6363
assetWriterPixelBufferInput = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput:assetWriterVideoInput, sourcePixelBufferAttributes:sourcePixelBufferAttributesDictionary)
6464
assetWriter.add(assetWriterVideoInput)
6565

66-
let (pipelineState, _) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"oneInputVertex", fragmentFunctionName:"passthroughFragment", operationName:"RenderView")
66+
let (pipelineState, _, _) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"oneInputVertex", fragmentFunctionName:"passthroughFragment", operationName:"RenderView")
6767
self.renderPipelineState = pipelineState
6868
}
6969

‎framework/Source/RenderView.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public class RenderView: MTKView, ImageConsumer {
2626

2727
self.device = sharedMetalRenderingDevice.device
2828

29-
let (pipelineState, _) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"oneInputVertex", fragmentFunctionName:"passthroughFragment", operationName:"RenderView")
29+
let (pipelineState, _, _) = generateRenderPipelineState(device:sharedMetalRenderingDevice, vertexFunctionName:"oneInputVertex", fragmentFunctionName:"passthroughFragment", operationName:"RenderView")
3030
self.renderPipelineState = pipelineState
3131

3232
enableSetNeedsDisplay = false

‎framework/Source/ShaderUniformSettings.swift‎

Lines changed: 11 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -2,40 +2,34 @@ import Foundation
22
import Metal
33

44
public class ShaderUniformSettings {
5-
private var uniformValues:[Float] = []
6-
private var uniformValueOffsets:[Int] = []
5+
let uniformLookupTable:[String:Int]
6+
private var uniformValues:[Float]
7+
private var uniformValueOffsets:[Int]
78
public var colorUniformsUseAlpha:Bool = false
89
let shaderUniformSettingsQueue = DispatchQueue(
910
label: "com.sunsetlakesoftware.GPUImage.shaderUniformSettings",
1011
attributes: [])
11-
let uniformLookupTable:[String:Int]
1212

13-
public init(uniformLookupTable:[String:(Int, MTLDataType)]) {
13+
public init(uniformLookupTable:[String:(Int, MTLStructMember)], bufferSize:Int) {
1414
var convertedLookupTable:[String:Int] = [:]
1515

16-
var orderedDatatypes = [MTLDataType](repeating:.float, count:uniformLookupTable.count)
16+
var orderedOffsets = [Int](repeating:0, count:uniformLookupTable.count)
1717

18-
for (key, value) in uniformLookupTable {
19-
let (index, dataType) = value
18+
for (key, uniform) in uniformLookupTable {
19+
let (index, structMember) = uniform
2020
convertedLookupTable[key] = index
21-
orderedDatatypes[index] = dataType
21+
orderedOffsets[index] = structMember.offset/4
2222
}
2323

2424
self.uniformLookupTable = convertedLookupTable
25-
26-
for dataType in orderedDatatypes {
27-
self.appendBufferSpace(for:dataType)
28-
}
25+
self.uniformValues = [Float](repeating:0.0, count:bufferSize/4)
26+
self.uniformValueOffsets = orderedOffsets
2927
}
3028

3129
public var usesAspectRatio:Bool { get { return self.uniformLookupTable["aspectRatio"] != nil } }
3230

3331
private func internalIndex(for index:Int) -> Int {
34-
if (index == 0) {
35-
return 0
36-
} else {
37-
return uniformValueOffsets[index - 1]
38-
}
32+
return uniformValueOffsets[index]
3933
}
4034

4135
// MARK: -
@@ -154,37 +148,6 @@ public class ShaderUniformSettings {
154148

155149
// MARK: -
156150
// MARK: Uniform buffer memory management
157-
158-
func appendBufferSpace(for dataType:MTLDataType) {
159-
let uniformSize:Int
160-
switch dataType {
161-
case .float: uniformSize = 1
162-
case .float2: uniformSize = 2
163-
case .float3: uniformSize = 4 // Hack to fix alignment issues
164-
case .float4: uniformSize = 4
165-
case .float3x3: uniformSize = 12
166-
case .float4x4: uniformSize = 16
167-
default: fatalError("Uniform data type of value: \(dataType.rawValue) not supported")
168-
}
169-
let blankValues = [Float](repeating:0.0, count:uniformSize)
170-
171-
let lastOffset = alignPackingForOffset(uniformSize:uniformSize, lastOffset:uniformValueOffsets.last ?? 0)
172-
uniformValues.append(contentsOf:blankValues)
173-
uniformValueOffsets.append(lastOffset + uniformSize)
174-
}
175-
176-
func alignPackingForOffset(uniformSize:Int, lastOffset:Int) -> Int {
177-
let floatAlignment = (lastOffset + uniformSize) % 4
178-
let previousFloatAlignment = lastOffset % 4
179-
if (uniformSize > 1) && (floatAlignment != 0) && (previousFloatAlignment != 0){
180-
let paddingToAlignment = 4 - floatAlignment
181-
uniformValues.append(contentsOf:[Float](repeating:0.0, count:paddingToAlignment))
182-
uniformValueOffsets[uniformValueOffsets.count - 1] = lastOffset + paddingToAlignment
183-
return lastOffset + paddingToAlignment
184-
} else {
185-
return lastOffset
186-
}
187-
}
188151

189152
public func restoreShaderSettings(renderEncoder:MTLRenderCommandEncoder) {
190153
shaderUniformSettingsQueue.sync {

‎framework/Source/YUVToRGBConversion.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ public let colorConversionMatrix709Default = Matrix3x3(rowMajorValues:[
2121
1.793, -0.533, 0.0,
2222
])
2323

24-
public func convertYUVToRGB(pipelineState:MTLRenderPipelineState, lookupTable:[String:(Int, MTLDataType)], luminanceTexture:Texture, chrominanceTexture:Texture, secondChrominanceTexture:Texture? = nil, resultTexture:Texture, colorConversionMatrix:Matrix3x3) {
25-
let uniformSettings = ShaderUniformSettings(uniformLookupTable:lookupTable)
24+
public func convertYUVToRGB(pipelineState:MTLRenderPipelineState, lookupTable:[String:(Int, MTLStructMember)], bufferSize:Int, luminanceTexture:Texture, chrominanceTexture:Texture, secondChrominanceTexture:Texture? = nil, resultTexture:Texture, colorConversionMatrix:Matrix3x3) {
25+
let uniformSettings = ShaderUniformSettings(uniformLookupTable:lookupTable, bufferSize:bufferSize)
2626
uniformSettings["colorConversionMatrix"] = colorConversionMatrix
2727

2828
guard let commandBuffer = sharedMetalRenderingDevice.commandQueue.makeCommandBuffer() else {return}

0 commit comments

Comments
 (0)