如何读取相机并以相机帧速率显示图像？
我想连续从网络摄像头读取图像（进行一些快速预处理），然后在窗口中显示图像。这应该以我的网络摄像头提供的帧速率（29 fps）运行。 OpenCV GUI 和 Tkinter GUI 似乎太慢，无法以这样的帧速率显示图像。这些显然是我实验中的瓶颈。即使没有预处理，图像的显示速度也不够快。我使用的是 MacBook Pro 2018。
这是我尝试过的。网络摄像头始终使用 OpenCV 读取：
这是代码：
单循环，OpenCV GUI：
import cv2
import time
def main():
cap = cv2.VideoCapture(0)
window_name = "FPS Single Loop"
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
start_time = time.time()
frames = 0
seconds_to_measure = 10
while start_time + seconds_to_measure > time.time():
success, img = cap.read()
img = img[:, ::-1] # mirror
time.sleep(0.01) # simulate some processing time
cv2.imshow(window_name, img)
cv2.waitKey(1)
frames = frames + 1
cv2.destroyAllWindows()
print(
f"Captured {frames} in {seconds_to_measure} seconds. FPS: {frames/seconds_to_measure}"
)
if __name__ == "__main__":
main()
Captured 121 in 10 seconds. FPS: 12.1
多线程，opencv 图形用户界面：
import logging
import time
from queue import Full, Queue
from threading import Thread, Event
import cv2
logger = logging.getLogger("VideoStream")
def setup_webcam_stream(src=0):
cap = cv2.VideoCapture(src)
width, height = (
cap.get(cv2.CAP_PROP_FRAME_WIDTH),
cap.get(cv2.CAP_PROP_FRAME_HEIGHT),
)
logger.info(f"Camera dimensions: {width, height}")
logger.info(f"Camera FPS: {cap.get(cv2.CAP_PROP_FPS)}")
grabbed, frame = cap.read() # Read once to init
if not grabbed:
raise IOError("Cannot read video stream.")
return cap
def video_stream_loop(video_stream: cv2.VideoCapture, queue: Queue, stop_event: Event):
while not stop_event.is_set():
try:
success, img = video_stream.read()
# We need a timeout here to not get stuck when no images are retrieved from the queue
queue.put(img, timeout=1)
except Full:
pass # try again with a newer frame
def processing_loop(input_queue: Queue, output_queue: Queue, stop_event: Event):
while not stop_event.is_set():
try:
img = input_queue.get()
img = img[:, ::-1] # mirror
time.sleep(0.01) # simulate some processing time
# We need a timeout here to not get stuck when no images are retrieved from the queue
output_queue.put(img, timeout=1)
except Full:
pass # try again with a newer frame
def main():
stream = setup_webcam_stream(0)
webcam_queue = Queue()
processed_queue = Queue()
stop_event = Event()
window_name = "FPS Multi Threading"
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
start_time = time.time()
frames = 0
seconds_to_measure = 10
try:
Thread(
target=video_stream_loop, args=[stream, webcam_queue, stop_event]
).start()
Thread(
target=processing_loop, args=[webcam_queue, processed_queue, stop_event]
).start()
while start_time + seconds_to_measure > time.time():
img = processed_queue.get()
cv2.imshow(window_name, img)
cv2.waitKey(1)
frames = frames + 1
finally:
stop_event.set()
cv2.destroyAllWindows()
print(
f"Captured {frames} frames in {seconds_to_measure} seconds. FPS: {frames/seconds_to_measure}"
)
print(f"Webcam queue: {webcam_queue.qsize()}")
print(f"Processed queue: {processed_queue.qsize()}")
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
main()
INFO:VideoStream:Camera dimensions: (1280.0, 720.0)
INFO:VideoStream:Camera FPS: 29.000049
Captured 209 frames in 10 seconds. FPS: 20.9
Webcam queue: 0
Processed queue: 82
在这里您可以看到第二个队列中剩余图像，其中图像被获取以显示它们。
当我取消注释这两行时：
cv2.imshow(window_name, img)
cv2.waitKey(1)
那么输出是：
INFO:VideoStream:Camera dimensions: (1280.0, 720.0)
INFO:VideoStream:Camera FPS: 29.000049
Captured 291 frames in 10 seconds. FPS: 29.1
Webcam queue: 0
Processed queue: 0
因此，它能够以网络摄像头的速度处理所有帧，而无需 GUI 显示它们。
多线程，Tkinter 图形用户界面：
import logging
import time
import tkinter
from queue import Full, Queue, Empty
from threading import Thread, Event
import PIL
from PIL import ImageTk
import cv2
logger = logging.getLogger("VideoStream")
def setup_webcam_stream(src=0):
cap = cv2.VideoCapture(src)
width, height = cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
logger.info(f"Camera dimensions: {width, height}")
logger.info(f"Camera FPS: {cap.get(cv2.CAP_PROP_FPS)}")
grabbed, frame = cap.read() # Read once to init
if not grabbed:
raise IOError("Cannot read video stream.")
return cap, width, height
def video_stream_loop(video_stream: cv2.VideoCapture, queue: Queue, stop_event: Event):
while not stop_event.is_set():
try:
success, img = video_stream.read()
# We need a timeout here to not get stuck when no images are retrieved from the queue
queue.put(img, timeout=1)
except Full:
pass # try again with a newer frame
def processing_loop(input_queue: Queue, output_queue: Queue, stop_event: Event):
while not stop_event.is_set():
try:
img = input_queue.get()
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img[:, ::-1] # mirror
time.sleep(0.01) # simulate some processing time
# We need a timeout here to not get stuck when no images are retrieved from the queue
output_queue.put(img, timeout=1)
except Full:
pass # try again with a newer frame
class App:
def __init__(self, window, window_title, image_queue: Queue, image_dimensions: tuple):
self.window = window
self.window.title(window_title)
self.image_queue = image_queue
# Create a canvas that can fit the above video source size
self.canvas = tkinter.Canvas(window, width=image_dimensions[0], height=image_dimensions[1])
self.canvas.pack()
# After it is called once, the update method will be automatically called every delay milliseconds
self.delay = 1
self.update()
self.window.mainloop()
def update(self):
try:
frame = self.image_queue.get(timeout=0.1) # Timeout to not block this method forever
self.photo = ImageTk.PhotoImage(image=PIL.Image.fromarray(frame))
self.canvas.create_image(0, 0, image=self.photo, anchor=tkinter.NW)
self.window.after(self.delay, self.update)
except Empty:
pass # try again next time
def main():
stream, width, height = setup_webcam_stream(0)
webcam_queue = Queue()
processed_queue = Queue()
stop_event = Event()
window_name = "FPS Multi Threading"
try:
Thread(target=video_stream_loop, args=[stream, webcam_queue, stop_event]).start()
Thread(target=processing_loop, args=[webcam_queue, processed_queue, stop_event]).start()
App(tkinter.Tk(), window_name, processed_queue, (width, height))
finally:
stop_event.set()
print(f"Webcam queue: {webcam_queue.qsize()}")
print(f"Processed queue: {processed_queue.qsize()}")
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
main()
INFO:VideoStream:Camera dimensions: (1280.0, 720.0)
INFO:VideoStream:Camera FPS: 29.000049
Webcam queue: 0
Processed queue: 968
关于这个答案我分享一些考虑相机帧率 VS 显示帧率以及一些演示的代码示例：
threading and queue有效地以相机支持的最接近的最大 fps 进行拍摄；对于遇到您的问题的任何人，以下是需要首先回答的几个重要问题：
The 相机帧率指的是相机的硬件能力。例如，ffmpeg告诉我，在 640x480 分辨率下，我的相机可以返回最小 15 fps，最大 30 fps，以及其他格式：
ffmpeg -list_devices true -f dshow -i dummy
ffmpeg -f dshow -list_options true -i video="HP HD Camera"
[dshow @ 00000220181cc600] vcodec=mjpeg min s=640x480 fps=15 max s=640x480 fps=30
[dshow @ 00000220181cc600] vcodec=mjpeg min s=320x180 fps=15 max s=320x180 fps=30
[dshow @ 00000220181cc600] vcodec=mjpeg min s=320x240 fps=15 max s=320x240 fps=30
[dshow @ 00000220181cc600] vcodec=mjpeg min s=424x240 fps=15 max s=424x240 fps=30
[dshow @ 00000220181cc600] vcodec=mjpeg min s=640x360 fps=15 max s=640x360 fps=30
[dshow @ 00000220181cc600] vcodec=mjpeg min s=848x480 fps=15 max s=848x480 fps=30
[dshow @ 00000220181cc600] vcodec=mjpeg min s=960x540 fps=15 max s=960x540 fps=30
[dshow @ 00000220181cc600] vcodec=mjpeg min s=1280x720 fps=15 max s=1280x720 fps=30
这里重要的认识是，尽管能够在内部捕获 30 fps，但不能保证应用程序能够在一秒钟内从相机中提取这 30 帧。以下各节阐明了其背后的原因。
The 显示帧率指的是每秒可以在一个窗口中绘制多少张图像。这个数字完全不受相机的限制，通常远高于相机的 fps。正如您稍后将看到的，可以创建每秒从相机提取 29 个图像并每秒绘制超过 300 次的应用程序。这意味着在从相机中拉出下一帧之前，会在窗口中多次绘制来自相机的相同图像。
以下应用程序简单演示了如何打印相机使用的默认设置（大小、fps）以及如何从中检索帧、将其显示在窗口中并计算正在渲染的 FPS 量：
import numpy as np
import cv2
import datetime
def main():
# create display window
cv2.namedWindow("webcam", cv2.WINDOW_NORMAL)
# initialize webcam capture object
cap = cv2.VideoCapture(0)
# retrieve properties of the capture object
cap_width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
cap_height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
cap_fps = cap.get(cv2.CAP_PROP_FPS)
fps_sleep = int(1000 / cap_fps)
print('* Capture width:', cap_width)
print('* Capture height:', cap_height)
print('* Capture FPS:', cap_fps, 'ideal wait time between frames:', fps_sleep, 'ms')
# initialize time and frame count variables
last_time = datetime.datetime.now()
frames = 0
# main loop: retrieves and displays a frame from the camera
while (True):
# blocks until the entire frame is read
success, img = cap.read()
frames += 1
# compute fps: current_time - last_time
delta_time = datetime.datetime.now() - last_time
elapsed_time = delta_time.total_seconds()
cur_fps = np.around(frames / elapsed_time, 1)
# draw FPS text and display image
cv2.putText(img, 'FPS: ' + str(cur_fps), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
cv2.imshow("webcam", img)
# wait 1ms for ESC to be pressed
key = cv2.waitKey(1)
if (key == 27):
break
# release resources
cv2.destroyAllWindows()
cap.release()
if __name__ == "__main__":
main()
Output:
* Capture width: 640.0
* Capture height: 480.0
* Capture FPS: 30.0 wait time between frames: 33 ms
如前所述，我的相机默认能够以 30 fps 的速度拍摄 640x480 的图像，尽管上面的循环非常简单，但我的相机显示帧率较低：我只能检索帧并以 28 或 29 fps 的速度显示它们，并且中间不执行任何自定义图像处理。这是怎么回事？
现实情况是，尽管循环看起来非常简单，但幕后发生的一些事情只花费了足够的处理时间，使得循环的一次迭代很难在不到 33 毫秒的时间内发生：
cap.read()对相机驱动程序执行 I/O 调用以提取新数据。此函数会阻止应用程序的执行，直到数据完全传输为止；cv2.imshow()，通常运行缓慢；cv2.waitKey(1)保持窗户打开所需的；所有这些操作，尽管很小，却使应用程序调用起来非常困难cap.read()，获取一个新帧并以 30 fps 精确显示。
您可以尝试多种方法来加快应用程序的速度，以便能够显示比相机驱动程序允许的更多的帧，并且这个帖子很好地覆盖它们。请记住这一点：你将无法捕获更多帧来自摄像头的内容比驱动程序所说的支持的内容要多。但是，您将能够显示更多帧.
threading例子。用于增加每秒显示的图像数量的方法之一依赖于threading包来创建一个单独的线程来连续从相机中提取帧。发生这种情况是因为应用程序的主循环没有被阻止cap.read()不再等待它返回新帧，从而增加每秒可以显示（或绘制）的帧数。
Note：此方法会在窗口上多次渲染同一图像，直到从相机检索到下一个图像。请记住，它甚至可能会在其内容仍在使用来自相机的新数据进行更新时绘制图像。
以下应用程序只是一个学术示例，我不建议将其作为生产代码，以增加窗口中每秒显示的帧数：
import numpy as np
import cv2
import datetime
from threading import Thread
# global variables
stop_thread = False # controls thread execution
img = None # stores the image retrieved by the camera
def start_capture_thread(cap):
global img, stop_thread
# continuously read fames from the camera
while True:
_, img = cap.read()
if (stop_thread):
break
def main():
global img, stop_thread
# create display window
cv2.namedWindow("webcam", cv2.WINDOW_NORMAL)
# initialize webcam capture object
cap = cv2.VideoCapture(0)
# retrieve properties of the capture object
cap_width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
cap_height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
cap_fps = cap.get(cv2.CAP_PROP_FPS)
fps_sleep = int(1000 / cap_fps)
print('* Capture width:', cap_width)
print('* Capture height:', cap_height)
print('* Capture FPS:', cap_fps, 'wait time between frames:', fps_sleep)
# start the capture thread: reads frames from the camera (non-stop) and stores the result in img
t = Thread(target=start_capture_thread, args=(cap,), daemon=True) # a deamon thread is killed when the application exits
t.start()
# initialize time and frame count variables
last_time = datetime.datetime.now()
frames = 0
cur_fps = 0
while (True):
# blocks until the entire frame is read
frames += 1
# measure runtime: current_time - last_time
delta_time = datetime.datetime.now() - last_time
elapsed_time = delta_time.total_seconds()
# compute fps but avoid division by zero
if (elapsed_time != 0):
cur_fps = np.around(frames / elapsed_time, 1)
# TODO: make a copy of the image and process it here if needed
# draw FPS text and display image
if (img is not None):
cv2.putText(img, 'FPS: ' + str(cur_fps), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
cv2.imshow("webcam", img)
# wait 1ms for ESC to be pressed
key = cv2.waitKey(1)
if (key == 27):
stop_thread = True
break
# release resources
cv2.destroyAllWindows()
cap.release()
if __name__ == "__main__":
main()
threading and queue例子。使用的问题queue就性能而言，您获得的结果取决于应用程序每秒可以从相机中提取多少帧。如果相机支持 30 fps 那么这就是您的应用程序might只要正在完成的图像处理操作很快即可。否则，显示的帧数（每秒）将会下降，并且队列的大小将缓慢增加，直到所有 RAM 内存耗尽。为了避免这个问题，请确保设置queueSize一个数字可以防止队列增长超出操作系统的处理能力。
以下代码是一个简单的实现，它创建了一个专用的thread从相机中抓取帧并将它们放入queue稍后由应用程序的主循环使用：
import numpy as np
import cv2
import datetime
import queue
from threading import Thread
# global variables
stop_thread = False # controls thread execution
def start_capture_thread(cap, queue):
global stop_thread
# continuously read fames from the camera
while True:
_, img = cap.read()
queue.put(img)
if (stop_thread):
break
def main():
global stop_thread
# create display window
cv2.namedWindow("webcam", cv2.WINDOW_NORMAL)
# initialize webcam capture object
cap = cv2.VideoCapture(0)
#cap = cv2.VideoCapture(0 + cv2.CAP_DSHOW)
# retrieve properties of the capture object
cap_width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
cap_height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
cap_fps = cap.get(cv2.CAP_PROP_FPS)
print('* Capture width:', cap_width)
print('* Capture height:', cap_height)
print('* Capture FPS:', cap_fps)
# create a queue
frames_queue = queue.Queue(maxsize=0)
# start the capture thread: reads frames from the camera (non-stop) and stores the result in img
t = Thread(target=start_capture_thread, args=(cap, frames_queue,), daemon=True) # a deamon thread is killed when the application exits
t.start()
# initialize time and frame count variables
last_time = datetime.datetime.now()
frames = 0
cur_fps = 0
while (True):
if (frames_queue.empty()):
continue
# blocks until the entire frame is read
frames += 1
# measure runtime: current_time - last_time
delta_time = datetime.datetime.now() - last_time
elapsed_time = delta_time.total_seconds()
# compute fps but avoid division by zero
if (elapsed_time != 0):
cur_fps = np.around(frames / elapsed_time, 1)
# retrieve an image from the queue
img = frames_queue.get()
# TODO: process the image here if needed
# draw FPS text and display image
if (img is not None):
cv2.putText(img, 'FPS: ' + str(cur_fps), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
cv2.imshow("webcam", img)
# wait 1ms for ESC to be pressed
key = cv2.waitKey(1)
if (key == 27):
stop_thread = True
break
# release resources
cv2.destroyAllWindows()
cap.release()
if __name__ == "__main__":
main()
之前我说过might这就是我的意思：即使我使用专用的thread从相机中提取帧并queue为了存储它们，显示的 fps 仍被限制为 29.3，而它本应为 30 fps。在这种情况下，我假设相机驱动程序或后端实现使用VideoCapture可以归咎于这个问题。在 Windows 上，默认使用的后端是MSMF.
可以强制VideoCapture通过在构造函数上传递正确的参数来使用不同的后端：
cap = cv2.VideoCapture(0 + cv2.CAP_DSHOW)
我的经验DShow太可怕了：返回CAP_PROP_FPS从相机是0并且显示的 FPS 被卡住了14。这只是一个示例，用于说明后端捕获驱动程序如何对相机捕获产生负面干扰。
但这是你可以探索的事情。也许在操作系统上使用不同的后端可以提供更好的结果。这是一个不错的OpenCV 视频 I/O 模块的高级概述列出了支持的后端：
Update
在这个答案的评论之一中，OP 在 Mac OS 上将 OpenCV 4.1 升级到 4.3，并观察到 FPS 渲染有明显的改进。看起来这是一个与以下相关的性能问题cv2.imshow().