我正在使用 Vision 来检测物体，然后得到[VNRecognizedObjectObservation]我在显示标准化矩形之前对其进行转换：
let transform = CGAffineTransform(scaleX: 1, y: -1).translatedBy(x: 0, y: -CGFloat(height))
VNImageRectForNormalizedRect(normalizedRect, width, height) // Displayed with SwiftUI, that's why I'm applying transform
.applying(transform)
宽度和高度来自 SwiftUI GeometryReader：
Image(...)
.resizable()
.scaledToFit()
.overlay {
GeometryReader { geometry in // ZStack and ForEach([VNRecognizedObjectObservation], id: \.uuid), then:
let calculatedRect = calculateRect(boundingBox, geometry)
Rectangle()
.frame(width: calculatedRect.width, height: calculatedRect.height)
.offset(x: calculatedRect.origin.x, y: calculatedRect.origin.y)
}
}
但问题是，即使在方形图像上，许多框的位置也不正确（尽管有些是准确的）。
这与模型无关，因为当我在 Xcode 模型预览部分尝试相同的图像（使用相同的 MLModel）时，它们具有非常准确的 BB。
我的应用程序中的示例图像：
Xcode 预览中的示例图像：
里面有这段代码ContentView.swift as a macOS SwiftUI项目同时拥有YOLOv3Tiny.ml模型在项目捆绑包中将产生相同的结果。
import SwiftUI
import Vision
import CoreML
class Detection: ObservableObject {
let imgURL = URL(string: "https://i.imgur.com/EqsxxTc.jpg")! // Xcode preview generates this: https://i.imgur.com/6IPNQ8b.png
@Published var objects: [VNRecognizedObjectObservation] = []
func getModel() -> VNCoreMLModel? {
if let modelURL = Bundle.main.url(forResource: "YOLOv3Tiny", withExtension: "mlmodelc") {
if let mlModel = try? MLModel(contentsOf: modelURL, configuration: MLModelConfiguration()) {
return try? VNCoreMLModel(for: mlModel)
}
}
return nil
}
func detect() async {
guard let model = getModel(), let tiff = NSImage(contentsOf: imgURL)?.tiffRepresentation else {
fatalError("Either YOLOv3Tiny.mlmodel is not in project bundle, or image failed to load.")
// YOLOv3Tiny: https://ml-assets.apple.com/coreml/models/Image/ObjectDetection/YOLOv3Tiny/YOLOv3Tiny.mlmodel
}
let request = VNCoreMLRequest(model: model) { (request, error) in
DispatchQueue.main.async {
self.objects = (request.results as? [VNRecognizedObjectObservation]) ?? []
}
}
try? VNImageRequestHandler(data: tiff).perform([request])
}
func deNormalize(_ rect: CGRect, _ geometry: GeometryProxy) -> CGRect {
let transform = CGAffineTransform(scaleX: 1, y: -1).translatedBy(x: 0, y: -CGFloat(geometry.size.height))
return VNImageRectForNormalizedRect(rect, Int(geometry.size.width), Int(geometry.size.height)).applying(transform)
}
}
struct ContentView: View {
@StateObject var detection = Detection()
var body: some View {
AsyncImage(url: detection.imgURL) { img in
img.resizable().scaledToFit().overlay {
GeometryReader { geometry in
ZStack {
ForEach(detection.objects, id: \.uuid) { object in
let rect = detection.deNormalize(object.boundingBox, geometry)
Rectangle()
.stroke(lineWidth: 2)
.foregroundColor(.red)
.frame(width: rect.width, height: rect.height)
.offset(x: rect.origin.x, y: rect.origin.y)
}
}
}
}
} placeholder: {
ProgressView()
}
.onAppear {
Task { await self.detection.detect() }
}
}
}
Edit:进一步的测试表明 VN 返回正确的位置，而我的deNormalize()函数还返回正确的位置和大小，因此它必须与 SwiftUI 相关。
Issue 1
GeometryReader使里面的所有东西都缩小到最小尺寸。
Add .border(Color.orange) to the ZStack你会看到类似我下面的内容。
您可以使用.frame(maxWidth: .infinity, maxHeight: .infinity)使ZStack拉伸以占据所有可用空间。
Issue 2
position vs offset.
offset通常从中心开始然后你offset按指定金额。
position更像是origin.
将此视图的中心定位在其父坐标空间中的指定坐标处。
Issue 3
调整中心位置与原点使用的左上角 (0, 0)。
Issue 4
The ZStack需要在X轴上翻转。
下面是完整的代码
import SwiftUI
import Vision
import CoreML
@MainActor
class Detection: ObservableObject {
//Moved file to assets
//let imgURL = URL(string: "https://i.imgur.com/EqsxxTc.jpg")! // Xcode preview generates this: https://i.imgur.com/6IPNQ8b.png
let imageName: String = "EqsxxTc"
@Published var objects: [VNRecognizedObjectObservation] = []
func getModel() throws -> VNCoreMLModel {
//Used model directly instead of loading from URL
let model = try YOLOv3Tiny(configuration: .init()).model
let mlModel = try VNCoreMLModel(for: model)
return mlModel
}
func detect() async throws {
let model = try getModel()
guard let tiff = NSImage(named: imageName)?.tiffRepresentation else {
// YOLOv3Tiny: https://ml-assets.apple.com/coreml/models/Image/ObjectDetection/YOLOv3Tiny/YOLOv3Tiny.mlmodel
//fatalError("Either YOLOv3Tiny.mlmodel is not in project bundle, or image failed to load.")
throw AppError.unableToLoadImage
}
//Completion handlers are not compatible with async/await you have to convert to a continuation.
self.objects = try await withCheckedThrowingContinuation { (cont: CheckedContinuation<[VNRecognizedObjectObservation], Error>) in
let request = VNCoreMLRequest(model: model) { (request, error) in
if let error = error{
cont.resume(throwing: error)
}else{
cont.resume(returning: (request.results as? [VNRecognizedObjectObservation]) ?? [])
}
}
do{
try VNImageRequestHandler(data: tiff).perform([request])
}catch{
cont.resume(throwing: error)
}
}
}
func deNormalize(_ rect: CGRect, _ geometry: GeometryProxy) -> CGRect {
return VNImageRectForNormalizedRect(rect, Int(geometry.size.width), Int(geometry.size.height))
}
}
struct ContentView: View {
@StateObject var detection = Detection()
var body: some View {
Image(detection.imageName)
.resizable()
.scaledToFit()
.overlay {
GeometryReader { geometry in
ZStack {
ForEach(detection.objects, id: \.uuid) { object in
let rect = detection.deNormalize(object.boundingBox, geometry)
Rectangle()
.stroke(lineWidth: 2)
.foregroundColor(.red)
.frame(width: rect.width, height: rect.height)
//Changed to position
//Adjusting for center vs leading origin
.position(x: rect.origin.x + rect.width/2, y: rect.origin.y + rect.height/2)
}
}
//Geometry reader makes the view shrink to its smallest size
.frame(maxWidth: .infinity, maxHeight: .infinity)
//Flip upside down
.rotation3DEffect(.degrees(180), axis: (x: 1, y: 0, z: 0))
}.border(Color.orange)
}
.task {
do{
try await self.detection.detect()
}catch{
//Always throw errors to the View so you can tell the user somehow. You don't want crashes or to leave the user waiting for something that has failed.
print(error)
}
}
}
}
struct ContentView_Previews: PreviewProvider {
static var previews: some View {
ContentView()
}
}
enum AppError: LocalizedError{
case cannotFindFile
case unableToLoadImage
}
正如您所注意到的，我还更改了其他一些内容，代码中有注释。