Write opencv frames into gstreamer rtsp server pipeline

classic Classic list List threaded Threaded
4 messages Options
Reply | Threaded
Open this post in threaded view
|

Write opencv frames into gstreamer rtsp server pipeline

WisdomPill
I'm trying to put opencv images into a gstreamer rtsp server in python.
I have some issue writing in the mediafactory, I'm new to gst-rtsp-server
ancd there's little documentation so I don't know exactly if I'm using the
right approach. I'm using a thread to start the MainLoop and I'm using the
main thread to create a buffer to push in the appsrc element of the
mediafactory pipeline. Am I using the right approach to obtain my objective?
Can anyone help me? My code is below:

    from threading import Thread
    from time import clock
   
    import cv2
    import gi
   
    gi.require_version('Gst', '1.0')
    gi.require_version('GstRtspServer', '1.0')
    from gi.repository import Gst, GstRtspServer, GObject
   
   
    class SensorFactory(GstRtspServer.RTSPMediaFactory):
        def __init__(self, **properties):
            super(SensorFactory, self).__init__(**properties)
            self.launch_string = 'appsrc !
video/x-raw,width=320,height=240,framerate=30/1 ' \
                                 '! videoconvert ! x264enc
speed-preset=ultrafast tune=zerolatency ' \
                                 '! rtph264pay config-interval=1 name=pay0
pt=96'
            self.pipeline = Gst.parse_launch(self.launch_string)
            self.appsrc = self.pipeline.get_child_by_index(4)
   
        def do_create_element(self, url):
            return self.pipeline
   
   
    class GstServer(GstRtspServer.RTSPServer):
        def __init__(self, **properties):
            super(GstServer, self).__init__(**properties)
            self.factory = SensorFactory()
            self.factory.set_shared(True)
            self.get_mount_points().add_factory("/test", self.factory)
            self.attach(None)
   
   
    GObject.threads_init()
    Gst.init(None)
   
    server = GstServer()
   
    loop = GObject.MainLoop()
    th = Thread(target=loop.run)
    th.start()
   
    print('Thread started')
   
    cap = cv2.VideoCapture(0)
   
    print(cap.isOpened())
   
    frame_number = 0
   
    fps = 30
    duration = 1 / fps
   
    timestamp = clock()
   
    while cap.isOpened():
        ret, frame = cap.read()
        if ret:
   
            print('Writing buffer')
   
            data = frame.tostring()
   
            buf = Gst.Buffer.new_allocate(None, len(data), None)
            buf.fill(0, data)
            buf.duration = fps
            timestamp = clock() - timestamp
            buf.pts = buf.dts = int(timestamp)
            buf.offset = frame_number
            frame_number += 1
            retval = server.factory.appsrc.emit('push-buffer', buf)
            print(retval)
   
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
   
    cap.release()

By the way I tried to copy the buffer creation from opencv source code but
I'm not sure I correctly trandlated the c++ code in python.



--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel
Reply | Threaded
Open this post in threaded view
|

Re: Write opencv frames into gstreamer rtsp server pipeline

Michael MacIntosh

Hello WisdomPill,

I would suggest checking out this example if you haven't done so already:

https://github.com/GStreamer/gst-rtsp-server/blob/master/examples/test-appsrc.c

It is in C and I apologize I am not familiar with the python interface.

Two things I notice is it doesn't look like you are setting the appsrc format to time.   By default it is bytes.  You can change that like this (C++):

gst_util_set_object_arg(G_OBJECT(video_appsrc), "format", "time");

Since you are setting the PTS on the frames, you need to do this.  You might be getting invalid/unexpected format errors because of this.

I would also recommend setting up pushing frames on the "media-configured" signal on the media factory (assuming you are using a live source).  You can stop pushing frames when you get GST_FLOW_NOT_OK when you push your frames.

This way you can start pushing frames only when a client connects.  And stop pushing when they disconnect.

It also does not look like you are setting the caps on the appsrc.  I am fairly certain you need to do this.

You might also want to post what log output and what errors you are getting so people can better help you.

Good luck, appsrc rtsp servers are pretty challenging.

Cheers,
Michael.


On 11/20/2017 8:13 AM, WisdomPill wrote:
I'm trying to put opencv images into a gstreamer rtsp server in python.
I have some issue writing in the mediafactory, I'm new to gst-rtsp-server
ancd there's little documentation so I don't know exactly if I'm using the
right approach. I'm using a thread to start the MainLoop and I'm using the
main thread to create a buffer to push in the appsrc element of the
mediafactory pipeline. Am I using the right approach to obtain my objective?
Can anyone help me? My code is below:

    from threading import Thread
    from time import clock
    
    import cv2
    import gi
    
    gi.require_version('Gst', '1.0')
    gi.require_version('GstRtspServer', '1.0')
    from gi.repository import Gst, GstRtspServer, GObject
    
    
    class SensorFactory(GstRtspServer.RTSPMediaFactory):
        def __init__(self, **properties):
            super(SensorFactory, self).__init__(**properties)
            self.launch_string = 'appsrc !
video/x-raw,width=320,height=240,framerate=30/1 ' \
                                 '! videoconvert ! x264enc
speed-preset=ultrafast tune=zerolatency ' \
                                 '! rtph264pay config-interval=1 name=pay0
pt=96'
            self.pipeline = Gst.parse_launch(self.launch_string)
            self.appsrc = self.pipeline.get_child_by_index(4)
    
        def do_create_element(self, url):
            return self.pipeline
    
    
    class GstServer(GstRtspServer.RTSPServer):
        def __init__(self, **properties):
            super(GstServer, self).__init__(**properties)
            self.factory = SensorFactory()
            self.factory.set_shared(True)
            self.get_mount_points().add_factory("/test", self.factory)
            self.attach(None)
    
    
    GObject.threads_init()
    Gst.init(None)
    
    server = GstServer()
    
    loop = GObject.MainLoop()
    th = Thread(target=loop.run)
    th.start()
    
    print('Thread started')
    
    cap = cv2.VideoCapture(0)
    
    print(cap.isOpened())
    
    frame_number = 0
    
    fps = 30
    duration = 1 / fps
    
    timestamp = clock()
    
    while cap.isOpened():
        ret, frame = cap.read()
        if ret:
    
            print('Writing buffer')
    
            data = frame.tostring()
    
            buf = Gst.Buffer.new_allocate(None, len(data), None)
            buf.fill(0, data)
            buf.duration = fps
            timestamp = clock() - timestamp
            buf.pts = buf.dts = int(timestamp)
            buf.offset = frame_number
            frame_number += 1
            retval = server.factory.appsrc.emit('push-buffer', buf)
            print(retval)
    
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break
    
    cap.release()

By the way I tried to copy the buffer creation from opencv source code but
I'm not sure I correctly trandlated the c++ code in python.



--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel




_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel
Reply | Threaded
Open this post in threaded view
|

Re: Write opencv frames into gstreamer rtsp server pipeline - baseline profile issue

WisdomPill
First and foremost, thank you for answering. You are very helpful.
I changed come stuff like you suggeted.
1) I started using need-data.
2) I set the caps on the appsrc
3) I changed the format

But I can display only the first frame, then I get the following error

x264 [error]: baseline profile doesn't support 4:4:4

My code is the following.

#!/usr/bin/env python3
from threading import Thread
from time import clock

import cv2
import gi

gi.require_version('Gst', '1.0')
gi.require_version('GstRtspServer', '1.0')
from gi.repository import Gst, GstRtspServer, GObject


class SensorFactory(GstRtspServer.RTSPMediaFactory):
    def __init__(self, **properties):
        super(SensorFactory, self).__init__(**properties)
        self.cap = cv2.VideoCapture(0)
        self.number_frames = 0
        self.fps = 30
        self.duration = 1 / self.fps * 1000
        self.timestamp = clock()
        self.launch_string = 'appsrc name=source
caps=video/x-raw,format=BGR,width=640,height=480,framerate=30/1 ' \
                             '! videoconvert ! x264enc
speed-preset=ultrafast tune=zerolatency ' \
                             '! rtph264pay config-interval=1 name=pay0
pt=96'
        self.pipeline = Gst.parse_launch(self.launch_string)
        self.appsrc = self.pipeline.get_child_by_name('source')
        self.appsrc.get_property('caps').fixate()
        self.appsrc.set_property('format', Gst.Format.TIME)
        self.bus = self.appsrc.get_bus()
        self.appsrc.connect('need-data', self.on_need_data)
        self.bus.connect('message::error', self.on_error)
        self.bus.connect('message::state-changed', self.on_status_changed)
        self.bus.connect('message::eos', self.on_eos)

    def on_need_data(self, src, lenght):
        if self.cap.isOpened():
            ret, frame = self.cap.read()
            if ret:
                data = frame.tostring()

                buf = Gst.Buffer.new_allocate(None, len(data), None)
                buf.fill(0, data)
                buf.duration = self.fps
                timestamp = self.number_frames * self.duration
                buf.pts = buf.dts = int(timestamp)
                buf.offset = self.number_frames
                self.number_frames += 1
                retval = server.factory.appsrc.emit('push-buffer', buf)
                if retval != Gst.FlowReturn.OK:
                    print(retval)

    def on_status_changed(self, bus, message):
        msg = message.parse_state_changed()
        print('status_changed message -> {}'.format(msg))

    def on_eos(self, bus, message):
        print('eos message -> {}'.format(message))

    def on_error(self, bus, message):
        print('error message -> {}'.format(message.parse_error().debug))

    def do_create_element(self, url):
        return self.pipeline


class GstServer(GstRtspServer.RTSPServer):
    def __init__(self, **properties):
        super(GstServer, self).__init__(**properties)
        self.factory = SensorFactory()
        self.factory.set_shared(True)
        self.get_mount_points().add_factory("/test", self.factory)
        self.attach(None)


GObject.threads_init()
Gst.init(None)

server = GstServer()

loop = GObject.MainLoop()
loop.run()





--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel
Reply | Threaded
Open this post in threaded view
|

Re: Write opencv frames into gstreamer rtsp server pipeline - baseline profile issue

WisdomPill
I found the solution, I was not setting the offset of the buffer correctly.
Here's the code for anyone who's facing the same problem or has a somewhat
similar one.

#!/usr/bin/env python3

import cv2
import gi

gi.require_version('Gst', '1.0')
gi.require_version('GstRtspServer', '1.0')
from gi.repository import Gst, GstRtspServer, GObject


class SensorFactory(GstRtspServer.RTSPMediaFactory):
    def __init__(self, **properties):
        super(SensorFactory, self).__init__(**properties)
        self.cap = cv2.VideoCapture(0)
        self.number_frames = 0
        self.fps = 30
        self.duration = 1 / self.fps * Gst.SECOND  # duration of a frame in
nanoseconds
        self.launch_string = 'appsrc name=source is-live=true block=true
format=GST_FORMAT_TIME ' \
                           
'caps=video/x-raw,format=BGR,width=640,height=480,framerate={}/1 ' \
                             '! videoconvert ! video/x-raw,format=I420 ' \
                             '! x264enc speed-preset=ultrafast
tune=zerolatency ' \
                             '! rtph264pay config-interval=1 name=pay0
pt=96'.format(self.fps)

    def on_need_data(self, src, lenght):
        if self.cap.isOpened():
            ret, frame = self.cap.read()
            if ret:
                data = frame.tostring()
                buf = Gst.Buffer.new_allocate(None, len(data), None)
                buf.fill(0, data)
                buf.duration = self.duration
                timestamp = self.number_frames * self.duration
                buf.pts = buf.dts = int(timestamp)
                buf.offset = timestamp
                self.number_frames += 1
                retval = src.emit('push-buffer', buf)
                print('pushed buffer, frame {}, duration {} ns, durations {}
s'.format(self.number_frames,
                                                                                     
self.duration,
                                                                                     
self.duration / Gst.SECOND))
                if retval != Gst.FlowReturn.OK:
                    print(retval)

    def do_create_element(self, url):
        return Gst.parse_launch(self.launch_string)

    def do_configure(self, rtsp_media):
        self.number_frames = 0
        appsrc = rtsp_media.get_element().get_child_by_name('source')
        appsrc.connect('need-data', self.on_need_data)


class GstServer(GstRtspServer.RTSPServer):
    def __init__(self, **properties):
        super(GstServer, self).__init__(**properties)
        self.factory = SensorFactory()
        self.factory.set_shared(True)
        self.get_mount_points().add_factory("/test", self.factory)
        self.attach(None)


GObject.threads_init()
Gst.init(None)

server = GstServer()

loop = GObject.MainLoop()
loop.run()




--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel