From 4958e5dd7ad6b1f4ce67e9242fc20cef48687b15 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Wed, 16 Dec 2020 16:36:27 +0000 Subject: [PATCH 001/339] rtptwcc: use interval-based feedback on RTCP thread This decouples the TWCC feedback reporting from the incoming RTP packet flow; instead, scheduling TWCC feedback reporting from the RTCP thread. --- .../gst/rtpmanager/rtpsession.c | 138 ++++++++++++++---- .../gst/rtpmanager/rtpsession.h | 1 + .../gst-plugins-good/gst/rtpmanager/rtptwcc.c | 39 ++++- .../gst-plugins-good/gst/rtpmanager/rtptwcc.h | 4 +- .../tests/check/elements/rtpsession.c | 103 ++++++++++++- 5 files changed, 247 insertions(+), 38 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 70c08774304..4fcd57222c5 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -758,6 +758,7 @@ rtp_session_init (RTPSession * sess) sess->last_rtcp_interval = GST_CLOCK_TIME_NONE; sess->next_early_rtcp_time = GST_CLOCK_TIME_NONE; + sess->next_twcc_rtcp_time = GST_CLOCK_TIME_NONE; sess->rtcp_feedback_retention_window = DEFAULT_RTCP_FEEDBACK_RETENTION_WINDOW; sess->rtcp_immediate_feedback_threshold = DEFAULT_RTCP_IMMEDIATE_FEEDBACK_THRESHOLD; @@ -1229,6 +1230,7 @@ rtp_session_reset (RTPSession * sess) sess->last_rtcp_send_time = GST_CLOCK_TIME_NONE; sess->last_rtcp_interval = GST_CLOCK_TIME_NONE; sess->next_early_rtcp_time = GST_CLOCK_TIME_NONE; + sess->next_twcc_rtcp_time = GST_CLOCK_TIME_NONE; sess->scheduled_bye = FALSE; /* reset session stats */ @@ -3745,9 +3747,17 @@ rtp_session_next_timeout (RTPSession * sess, GstClockTime current_time) } else { if (sess->first_rtcp) { GST_DEBUG ("first RTCP packet"); - /* we are called for the first time */ - interval = calculate_rtcp_interval (sess, FALSE, TRUE); - sess->last_rtcp_interval = interval; + /* We have yet to generate our first rtcp packet. Compute the initial + * interval when the next check time is changing. Otherwise, there is + * nothing to do (as the next check time is still in the future and + * computing a new interval will cause the first rtcp to be delayed). + * When interval-based TWCC is enabled, there can be multiple timeouts + * before the initial rtcp time (and thus, without this conditional, + * the initial rtcp will be delayed indefinitely). */ + if (result != sess->next_rtcp_check_time) { + interval = calculate_rtcp_interval (sess, FALSE, TRUE); + sess->last_rtcp_interval = interval; + } } else if (sess->next_rtcp_check_time < current_time) { GST_DEBUG ("old check time expired, getting new timeout"); /* get a new timeout when we need to */ @@ -3782,6 +3792,18 @@ rtp_session_next_timeout (RTPSession * sess, GstClockTime current_time) early_exit: + sess->next_twcc_rtcp_time = rtp_twcc_manager_get_next_timeout (sess->twcc, + current_time); + if (GST_CLOCK_TIME_IS_VALID (sess->next_twcc_rtcp_time)) { + /* Take min(twcc-time, result) */ + if (!GST_CLOCK_TIME_IS_VALID (result) || sess->next_twcc_rtcp_time < result) { + GST_DEBUG ("Using twcc time %" GST_TIME_FORMAT + " instead of %" GST_TIME_FORMAT, + GST_TIME_ARGS (sess->next_twcc_rtcp_time), GST_TIME_ARGS (result)); + result = sess->next_twcc_rtcp_time; + } + } + GST_DEBUG ("current time: %" GST_TIME_FORMAT ", next time: %" GST_TIME_FORMAT, GST_TIME_ARGS (current_time), GST_TIME_ARGS (result)); @@ -4510,7 +4532,8 @@ generate_twcc (const gchar * key, RTPSource * source, ReportData * data) GST_DEBUG ("generating TWCC feedback for source %08x", source->ssrc); - while ((buf = rtp_twcc_manager_get_feedback (sess->twcc, source->ssrc))) { + while ((buf = rtp_twcc_manager_get_feedback (sess->twcc, source->ssrc, + data->current_time))) { ReportOutput *output = g_new (ReportOutput, 1); output->source = g_object_ref (source); output->is_bye = FALSE; @@ -4646,6 +4669,32 @@ rtp_session_are_all_sources_bye (RTPSession * sess) return TRUE; } +static gboolean +awoken_for_twcc (RTPSession * sess, GstClockTime current_time) +{ + if (!GST_CLOCK_TIME_IS_VALID (rtp_twcc_manager_get_feedback_interval + (sess->twcc))) { + /* Interval-based TWCC is disabled, so this cannot be a TWCC wake */ + return FALSE; + } + + if (GST_CLOCK_TIME_IS_VALID (sess->next_early_rtcp_time) && + sess->next_early_rtcp_time <= current_time) { + /* Early time is before now, so this may be a non-TWCC wake */ + return FALSE; + } + + if (GST_CLOCK_TIME_IS_VALID (sess->next_rtcp_check_time) && + sess->next_rtcp_check_time <= current_time) { + /* Check time is before now, so this may be a non-TWCC wake */ + return FALSE; + } + + /* Interval-based TWCC is enabled, and neither early nor check times + * are before now, so this must be a TWCC wake */ + return TRUE; +} + /** * rtp_session_on_timeout: * @sess: an #RTPSession @@ -4673,6 +4722,7 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, GHashTable *table_copy; ReportOutput *output; gboolean all_empty = FALSE; + gboolean twcc_only = FALSE; g_return_val_if_fail (RTP_IS_SESSION (sess), GST_FLOW_ERROR); @@ -4691,6 +4741,7 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, g_queue_init (&data.output); RTP_SESSION_LOCK (sess); + /* get a new interval, we need this for various cleanups etc */ data.interval = calculate_rtcp_interval (sess, TRUE, sess->first_rtcp); @@ -4711,32 +4762,54 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, g_object_unref (source); } - sess->conflicting_addresses = - timeout_conflicting_addresses (sess->conflicting_addresses, current_time); + twcc_only = awoken_for_twcc (sess, current_time); + if (!twcc_only) { + sess->conflicting_addresses = + timeout_conflicting_addresses (sess->conflicting_addresses, + current_time); - /* Make a local copy of the hashtable. We need to do this because the - * cleanup stage below releases the session lock. */ - table_copy = g_hash_table_new_full (NULL, NULL, NULL, - (GDestroyNotify) g_object_unref); - g_hash_table_foreach (sess->ssrcs[sess->mask_idx], - (GHFunc) clone_ssrcs_hashtable, table_copy); + /* Make a local copy of the hashtable. We need to do this because the + * update stage below releases the session lock. */ + table_copy = g_hash_table_new_full (NULL, NULL, NULL, + (GDestroyNotify) g_object_unref); + g_hash_table_foreach (sess->ssrcs[sess->mask_idx], + (GHFunc) clone_ssrcs_hashtable, table_copy); - /* Clean up the session, mark the source for removing, this might release the - * session lock. */ - g_hash_table_foreach (table_copy, (GHFunc) session_cleanup, &data); - g_hash_table_destroy (table_copy); + /* Clean up the session, mark the source for removing and update clock-rate, + * this might release the session lock. */ + g_hash_table_foreach (table_copy, (GHFunc) session_cleanup, &data); + g_hash_table_destroy (table_copy); - /* Now remove the marked sources */ - g_hash_table_foreach_remove (sess->ssrcs[sess->mask_idx], - (GHRFunc) remove_closing_sources, &data); + /* Now remove the marked sources */ + g_hash_table_foreach_remove (sess->ssrcs[sess->mask_idx], + (GHRFunc) remove_closing_sources, &data); - /* update point-to-point status */ - session_update_ptp (sess); + /* update point-to-point status */ + session_update_ptp (sess); + } + + /* generate interval-based twcc feedback */ + if (GST_CLOCK_TIME_IS_VALID (rtp_twcc_manager_get_feedback_interval + (sess->twcc))) { + GST_DEBUG ("interval-based twcc. now: %" GST_TIME_FORMAT " twcc-time: %" + GST_TIME_FORMAT, GST_TIME_ARGS (current_time), + GST_TIME_ARGS (sess->next_twcc_rtcp_time)); + if (GST_CLOCK_TIME_IS_VALID (sess->next_twcc_rtcp_time) + && sess->next_twcc_rtcp_time <= current_time) { + GST_DEBUG ("generating twcc"); + g_hash_table_foreach (sess->ssrcs[sess->mask_idx], + (GHFunc) generate_twcc, &data); + sess->next_twcc_rtcp_time = GST_CLOCK_TIME_NONE; + } + } /* see if we need to generate SR or RR packets */ if (!is_rtcp_time (sess, current_time, &data)) goto done; + /* Cannot get here if TWCC only */ + g_assert (!twcc_only); + /* check if all the buffers are empty after generation */ all_empty = TRUE; @@ -4755,7 +4828,12 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, * session lock. */ g_hash_table_foreach (table_copy, (GHFunc) generate_rtcp, &data); - g_hash_table_foreach (table_copy, (GHFunc) generate_twcc, &data); + /* add twcc feedback if not using interval-based feedback */ + if (!GST_CLOCK_TIME_IS_VALID (rtp_twcc_manager_get_feedback_interval + (sess->twcc))) { + GST_DEBUG ("generating irregular twcc"); + g_hash_table_foreach (table_copy, (GHFunc) generate_twcc, &data); + } /* update the generation for all the sources that have been reported */ g_hash_table_foreach (table_copy, (GHFunc) update_generation, &data); @@ -4786,7 +4864,9 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, RTP_SESSION_UNLOCK (sess); /* notify about updated statistics */ - g_object_notify_by_pspec (G_OBJECT (sess), properties[PROP_STATS]); + if (!twcc_only) { + g_object_notify_by_pspec (G_OBJECT (sess), properties[PROP_STATS]); + } /* push out the RTCP packets */ while ((output = g_queue_pop_head (&data.output))) { @@ -4839,11 +4919,13 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, if (all_empty) GST_ERROR ("generated empty RTCP messages for all the sources"); - /* schedule remaining nacks */ - RTP_SESSION_LOCK (sess); - g_hash_table_foreach (sess->ssrcs[sess->mask_idx], - (GHFunc) schedule_remaining_nacks, &data); - RTP_SESSION_UNLOCK (sess); + if (!twcc_only) { + /* schedule remaining nacks */ + RTP_SESSION_LOCK (sess); + g_hash_table_foreach (sess->ssrcs[sess->mask_idx], + (GHFunc) schedule_remaining_nacks, &data); + RTP_SESSION_UNLOCK (sess); + } return result; } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h index e48810ed0ef..4afa12cd344 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h @@ -279,6 +279,7 @@ struct _RTPSession { gboolean allow_early; GstClockTime next_early_rtcp_time; + GstClockTime next_twcc_rtcp_time; gboolean sr_req_pending; gboolean scheduled_bye; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c index 2b642bff6bb..f92a2b58b6e 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c @@ -312,6 +312,20 @@ rtp_twcc_manager_get_recv_twcc_seqnum (RTPTWCCManager * twcc, return val; } +GstClockTime +rtp_twcc_manager_get_next_timeout (RTPTWCCManager * twcc, + GstClockTime current_time) +{ + if (GST_CLOCK_TIME_IS_VALID (twcc->feedback_interval)) { + if (!GST_CLOCK_TIME_IS_VALID (twcc->next_feedback_send_time)) { + /* First time through: initialise feedback time */ + twcc->next_feedback_send_time = current_time + twcc->feedback_interval; + } + return twcc->next_feedback_send_time; + } + return GST_CLOCK_TIME_NONE; +} + static gint _twcc_seqnum_sort (gconstpointer a, gconstpointer b) { @@ -827,7 +841,7 @@ rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) if (pinfo->running_time >= twcc->next_feedback_send_time) { GST_LOG ("Generating feedback : Exceeded feedback interval %" GST_TIME_FORMAT, GST_TIME_ARGS (twcc->feedback_interval)); - rtp_twcc_manager_create_feedback (twcc); + rtp_twcc_manager_create_feedback_unlocked (twcc); send_feedback = TRUE; while (pinfo->running_time >= twcc->next_feedback_send_time) @@ -836,7 +850,7 @@ rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) } else if (pinfo->marker || _many_packets_some_lost (twcc, seqnum)) { GST_LOG ("Generating feedback because of %s", pinfo->marker ? "marker packet" : "many packets some lost"); - rtp_twcc_manager_create_feedback (twcc); + rtp_twcc_manager_create_feedback_unlocked (twcc); send_feedback = TRUE; twcc->packet_count_no_marker = 0; @@ -857,9 +871,28 @@ _change_rtcp_fb_sender_ssrc (GstBuffer * buf, guint32 sender_ssrc) } GstBuffer * -rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint sender_ssrc) +rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint sender_ssrc, + GstClockTime current_time) { GstBuffer *buf; + + GST_DEBUG ("considering twcc. now: %" GST_TIME_FORMAT + " twcc-time: %" GST_TIME_FORMAT " packets: %d", + GST_TIME_ARGS (current_time), + GST_TIME_ARGS (twcc->next_feedback_send_time), twcc->recv_packets->len); + + if (GST_CLOCK_TIME_IS_VALID (twcc->feedback_interval) && + GST_CLOCK_TIME_IS_VALID (twcc->next_feedback_send_time) && + twcc->next_feedback_send_time <= current_time) { + /* Sending on a fixed interval: compute next send time */ + while (twcc->next_feedback_send_time <= current_time) + twcc->next_feedback_send_time += twcc->feedback_interval; + + /* Generate feedback, if there is some to send */ + if (twcc->recv_packets->len > 0) + rtp_twcc_manager_create_feedback (twcc); + } + buf = g_queue_pop_head (twcc->rtcp_buffers); if (buf && twcc->recv_sender_ssrc != sender_ssrc) { diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h index a826e9a4c9a..bc8af8c4a19 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h @@ -63,6 +63,8 @@ void rtp_twcc_manager_set_mtu (RTPTWCCManager * twcc, guint mtu); void rtp_twcc_manager_set_feedback_interval (RTPTWCCManager * twcc, GstClockTime feedback_interval); GstClockTime rtp_twcc_manager_get_feedback_interval (RTPTWCCManager * twcc); +GstClockTime rtp_twcc_manager_get_next_timeout (RTPTWCCManager * twcc, + GstClockTime current_time); gboolean rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo); @@ -70,7 +72,7 @@ void rtp_twcc_manager_send_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo); GstBuffer * rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, - guint32 sender_ssrc); + guint32 sender_ssrc, GstClockTime current_time); GArray * rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, guint8 * fci_data, guint fci_length); diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 60316fc9e18..23de2d8a5be 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -4097,7 +4097,7 @@ typedef struct static TWCCFeedbackIntervalCtx test_twcc_feedback_interval_ctx[] = { {50 * GST_MSECOND, 21, 10 * GST_MSECOND, 4}, {50 * GST_MSECOND, 16, 7 * GST_MSECOND, 2}, - {50 * GST_MSECOND, 16, 66 * GST_MSECOND, 15}, + {50 * GST_MSECOND, 16, 66 * GST_MSECOND, 16}, {50 * GST_MSECOND, 15, 33 * GST_MSECOND, 9}, }; @@ -4106,23 +4106,78 @@ GST_START_TEST (test_twcc_feedback_interval) SessionHarness *h = session_harness_new (); GstBuffer *buf; TWCCFeedbackIntervalCtx *ctx = &test_twcc_feedback_interval_ctx[__i__]; + GstClockTime ts, next_feedback_time, last_twcc_time, inter_arrival_sum; + GstClockTime expected_inter_arrival_sum; + guint feedback_received = 0; session_harness_set_twcc_recv_ext_id (h, TEST_TWCC_EXT_ID); g_object_set (h->internal_session, "twcc-feedback-interval", ctx->interval, NULL); + ts = gst_clock_get_time (GST_CLOCK_CAST (h->testclock)); + next_feedback_time = ts + ctx->interval; + last_twcc_time = GST_CLOCK_TIME_NONE; + inter_arrival_sum = 0; + for (guint i = 0; i < ctx->num_packets; i++) { - GstClockTime ts = i * ctx->ts_delta; - gst_test_clock_set_time ((h->testclock), ts); + /* Advance to last TWCC interval before ts */ + while (next_feedback_time < ts) { + session_harness_crank_clock (h); + gst_test_clock_wait_for_next_pending_id (h->testclock, NULL); + next_feedback_time += ctx->interval; + } + + /* Advance time, if we haven't already gone past it */ + if (ts > gst_clock_get_time (GST_CLOCK_CAST (h->testclock))) + gst_test_clock_set_time ((h->testclock), ts); + + /* Push recv RTP */ fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h, generate_twcc_recv_buffer (i, ts, FALSE))); + + if (next_feedback_time <= ts + ctx->ts_delta) { + GstClockTime now; + + /* We expect a feedback report */ + buf = session_harness_produce_twcc (h); + gst_buffer_unref (buf); + + /* Time will have advanced to the feedback send time */ + now = gst_clock_get_time (GST_CLOCK_CAST (h->testclock)); + if (GST_CLOCK_TIME_IS_VALID (last_twcc_time)) + inter_arrival_sum += (now - last_twcc_time); + last_twcc_time = now; + feedback_received += 1; + + /* Compute next expected feedback time */ + next_feedback_time += ctx->interval; + } + + ts += ctx->ts_delta; } - for (guint i = 0; i < ctx->num_feedback; i++) { - buf = session_harness_produce_twcc (h); - gst_buffer_unref (buf); + /* Compute expected inter-arrival sum for feedback reports */ + if (ctx->ts_delta <= ctx->interval) { + /* Easy case: delta between packets is less than the feedback interval. + * In this case we expect the feedback reports to be continuous and + * spaced at the specified interval + */ + expected_inter_arrival_sum = (ctx->num_feedback - 1) * ctx->interval; + } else { + /* Inter-packet delta is more than the feedback interval. + * In this case we expect gaps in the feedback stream (because we do + * not send empty feedback reports) and thus the sum of deltas between + * feedback reports must be equal to the next multiple of the feedback + * interval after the time at which the last packet is sent + */ + expected_inter_arrival_sum = ((ctx->num_feedback - 1) * ctx->ts_delta) / + ctx->interval * ctx->interval; } + /* Ensure we got the reports we expected, spaced correctly */ + g_assert_cmpint (feedback_received, ==, ctx->num_feedback); + g_assert_cmpint (inter_arrival_sum, ==, expected_inter_arrival_sum); + session_harness_free (h); } @@ -4303,6 +4358,41 @@ GST_START_TEST (test_twcc_run_length_min) GST_END_TEST; +static void +set_min_rtcp_interval_on_new_sender_ssrc (GObject * session, + G_GNUC_UNUSED GObject * source) +{ + /* The value is irrelevant here: we want the side-effect of + * causing next_rtcp_check_time to be modified */ + g_object_set (session, "rtcp-min-interval", 5 * GST_SECOND, NULL); +} + +GST_START_TEST (test_twcc_feedback_interval_new_internal_source) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + + g_object_set (h->internal_session, + "twcc-feedback-interval", 50 * GST_MSECOND, NULL); + + g_object_set (h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); + g_signal_connect (h->internal_session, "on-new-sender-ssrc", + G_CALLBACK (set_min_rtcp_interval_on_new_sender_ssrc), NULL); + + /* Receive a RTP buffer from the wire */ + fail_unless_equals_int (GST_FLOW_OK, + session_harness_recv_rtp (h, generate_test_buffer (0, 0x12345678))); + + /* Wait for first regular RTCP to be sent */ + session_harness_produce_rtcp (h, 1); + buf = session_harness_pull_rtcp (h); + fail_unless (gst_rtcp_buffer_validate (buf)); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; static Suite * rtpsession_suite (void) @@ -4380,6 +4470,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_twcc_multiple_payloads_below_window); tcase_add_loop_test (tc_chain, test_twcc_feedback_interval, 0, G_N_ELEMENTS (test_twcc_feedback_interval_ctx)); + tcase_add_test (tc_chain, test_twcc_feedback_interval_new_internal_source); tcase_add_test (tc_chain, test_twcc_feedback_count_wrap); tcase_add_test (tc_chain, test_twcc_feedback_old_seqnum); From 2ebb3b435cba46d4996f56bafc844ff18ffa41d1 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 28 Feb 2020 22:43:59 +0100 Subject: [PATCH 002/339] rtptwcc: introduce and implement the GstTxFeedback interface The idea is that for TWCC you want to know as precisely as possible when a buffer you are sending leaves your system By using this method, sinks can then report back their most accurate estimate of when this actually happened. The implementation is a new GstTxFeedback meta with an interface that can be implemented by anyone interested in this accurate transmit time. The patch includes a basic implementation for the multiudpsink. multiudpsink: initial attempt at using the GstTxFeedback meta --- .../gst-plugins-good/gst/rtpmanager/rtptwcc.c | 48 ++++- .../gst/udp/gstmultiudpsink.c | 26 +++ subprojects/gstreamer/libs/gst/net/gstnet.h | 1 + .../gstreamer/libs/gst/net/gsttxfeedback.c | 170 ++++++++++++++++++ .../gstreamer/libs/gst/net/gsttxfeedback.h | 101 +++++++++++ .../gstreamer/libs/gst/net/meson.build | 2 + subprojects/gstreamer/libs/gst/net/net.h | 1 + .../tests/check/libs/gsttxfeedback.c | 116 ++++++++++++ subprojects/gstreamer/tests/check/meson.build | 1 + 9 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 subprojects/gstreamer/libs/gst/net/gsttxfeedback.c create mode 100644 subprojects/gstreamer/libs/gst/net/gsttxfeedback.h create mode 100644 subprojects/gstreamer/tests/check/libs/gsttxfeedback.c diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c index f92a2b58b6e..3b702ef9384 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "gstrtputils.h" @@ -105,7 +106,19 @@ struct _RTPTWCCManager GstClockTime feedback_interval; }; -G_DEFINE_TYPE (RTPTWCCManager, rtp_twcc_manager, G_TYPE_OBJECT); + +static void rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, + guint64 buffer_id, GstClockTime ts); + +static void +_tx_feedback_init (gpointer g_iface, G_GNUC_UNUSED gpointer iface_data) +{ + GstTxFeedbackInterface *iface = g_iface; + iface->tx_feedback = rtp_twcc_manager_tx_feedback; +} + +G_DEFINE_TYPE_WITH_CODE (RTPTWCCManager, rtp_twcc_manager, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (GST_TYPE_TX_FEEDBACK, _tx_feedback_init)); static void rtp_twcc_manager_init (RTPTWCCManager * twcc) @@ -262,6 +275,9 @@ _set_twcc_seqnum_data (RTPTWCCManager * twcc, RTPPacketInfo * pinfo, sent_packet_init (&packet, seqnum, pinfo, &rtp); g_array_append_val (twcc->sent_packets, packet); + gst_buffer_add_tx_feedback_meta (pinfo->data, seqnum, + GST_TX_FEEDBACK_CAST (twcc)); + GST_LOG ("Send: twcc-seqnum: %u, pt: %u, marker: %d, len: %u, ts: %" GST_TIME_FORMAT, seqnum, packet.pt, pinfo->marker, packet.size, GST_TIME_ARGS (pinfo->current_time)); @@ -912,6 +928,36 @@ rtp_twcc_manager_send_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) rtp_twcc_manager_set_send_twcc_seqnum (twcc, pinfo); } +static void +rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, guint64 buffer_id, + GstClockTime ts) +{ + RTPTWCCManager *twcc = RTP_TWCC_MANAGER_CAST (parent); + guint16 seqnum = (guint16) buffer_id; + SentPacket *first = NULL; + SentPacket *pkt = NULL; + guint idx; + + first = &g_array_index (twcc->sent_packets, SentPacket, 0); + if (first == NULL) { + GST_WARNING ("Received a tx-feedback without having sent any packets?!?"); + return; + } + + idx = seqnum - first->seqnum; + + if (idx < twcc->sent_packets->len) { + pkt = &g_array_index (twcc->sent_packets, SentPacket, idx); + if (pkt && pkt->seqnum == seqnum) { + pkt->socket_ts = ts; + GST_LOG ("packet #%u, setting socket-ts %" GST_TIME_FORMAT, + seqnum, GST_TIME_ARGS (ts)); + } + } else { + GST_WARNING ("Unable to update send-time for twcc-seqnum #%u", seqnum); + } +} + static void _add_twcc_packet (GArray * twcc_packets, guint16 seqnum, guint status) { diff --git a/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c b/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c index 527beeac870..1f3dca6fced 100644 --- a/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c +++ b/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c @@ -714,6 +714,28 @@ gst_multiudpsink_send_messages (GstMultiUDPSink * sink, GSocket * socket, return GST_FLOW_OK; } +static void +_set_time_on_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, + guint num_buffers) +{ + GstClock *clock = GST_ELEMENT_CLOCK (sink); + GstClockTime base_time = GST_ELEMENT_CAST (sink)->base_time; + GstClockTime now; + guint i; + + if (clock == NULL) + return; + + now = gst_clock_get_time (clock) - base_time; + + for (i = 0; i < num_buffers; i++) { + GstTxFeedbackMeta *meta = gst_buffer_get_tx_feedback_meta (buffers[i]); + if (meta) { + gst_tx_feedback_meta_set_tx_time (meta, now); + } + } +} + static GstFlowReturn gst_multiudpsink_render_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, guint num_buffers, guint8 * mem_nums, guint total_mem_num) @@ -832,6 +854,9 @@ gst_multiudpsink_render_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, if (flow_ret != GST_FLOW_OK) goto cancelled; + + _set_time_on_buffers (sink, buffers, num_buffers); + /* now update stats */ g_mutex_lock (&sink->client_lock); @@ -843,6 +868,7 @@ gst_multiudpsink_render_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, bytes_sent = msgs[i * num_buffers + j].bytes_sent; + client->bytes_sent += bytes_sent; client->packets_sent++; sink->bytes_served += bytes_sent; diff --git a/subprojects/gstreamer/libs/gst/net/gstnet.h b/subprojects/gstreamer/libs/gst/net/gstnet.h index 28173809c27..2eebe165a90 100644 --- a/subprojects/gstreamer/libs/gst/net/gstnet.h +++ b/subprojects/gstreamer/libs/gst/net/gstnet.h @@ -27,5 +27,6 @@ #include #include #include +#include #endif /* __GST_NET_H__ */ diff --git a/subprojects/gstreamer/libs/gst/net/gsttxfeedback.c b/subprojects/gstreamer/libs/gst/net/gsttxfeedback.c new file mode 100644 index 00000000000..cbd9d3f1f6f --- /dev/null +++ b/subprojects/gstreamer/libs/gst/net/gsttxfeedback.c @@ -0,0 +1,170 @@ +/* GStreamer + * Copyright (C) <2020> Havard Graff + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "gsttxfeedback.h" + +G_DEFINE_INTERFACE (GstTxFeedback, gst_tx_feedback, 0); + +static void +gst_tx_feedback_default_init (GstTxFeedbackInterface * iface) +{ + /* default virtual functions */ + iface->tx_feedback = NULL; +} + +static void +gst_tx_feedback_send_timestamp (GstTxFeedback * parent, + guint64 buffer_id, GstClockTime ts) +{ + g_return_if_fail (GST_IS_TX_FEEDBACK (parent)); + + g_assert (GST_TX_FEEDBACK_GET_INTERFACE (parent)->tx_feedback); + + GST_TX_FEEDBACK_GET_INTERFACE (parent)->tx_feedback (parent, buffer_id, ts); +} + +static gboolean +tx_feedback_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) +{ + GstTxFeedbackMeta *nmeta = (GstTxFeedbackMeta *) meta; + + nmeta->buffer_id = 0; + nmeta->feedback = NULL; + + return TRUE; +} + +static gboolean +tx_feedback_meta_transform (GstBuffer * transbuf, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + GstTxFeedbackMeta *smeta, *dmeta; + smeta = (GstTxFeedbackMeta *) meta; + + /* we always copy no matter what transform */ + dmeta = gst_buffer_add_tx_feedback_meta (transbuf, + smeta->buffer_id, smeta->feedback); + if (!dmeta) + return FALSE; + + return TRUE; +} + +static void +tx_feedback_meta_free (GstMeta * meta, GstBuffer * buffer) +{ + GstTxFeedbackMeta *nmeta = (GstTxFeedbackMeta *) meta; + gst_clear_object (&nmeta->feedback); +} + +GType +gst_tx_feedback_meta_api_get_type (void) +{ + static GType type; + static const gchar *tags[] = { "origin", NULL }; + + if (g_once_init_enter (&type)) { + GType _type = gst_meta_api_type_register ("GstTxFeedbackMetaAPI", tags); + g_once_init_leave (&type, _type); + } + return type; +} + +const GstMetaInfo * +gst_tx_feedback_meta_get_info (void) +{ + static const GstMetaInfo *meta_info = NULL; + + if (g_once_init_enter ((GstMetaInfo **) & meta_info)) { + const GstMetaInfo *mi = gst_meta_register (GST_TX_FEEDBACK_META_API_TYPE, + "GstTxFeedbackMeta", + sizeof (GstTxFeedbackMeta), + tx_feedback_meta_init, + tx_feedback_meta_free, tx_feedback_meta_transform); + g_once_init_leave ((GstMetaInfo **) & meta_info, (GstMetaInfo *) mi); + } + return meta_info; +} + +/** + * gst_tx_feedback_meta_set_tx_time: + * @meta: a #GstTxFeedbackMeta + * @ts: a @GstClockTime with the transmit time + * + * Notifies the interface about the buffer-id being transmitted at time ts + * + * Since: 1.22 + */ +void +gst_tx_feedback_meta_set_tx_time (GstTxFeedbackMeta * meta, GstClockTime ts) +{ + gst_tx_feedback_send_timestamp (meta->feedback, meta->buffer_id, ts); +} + +/** + * gst_buffer_add_tx_feedback_meta: + * @buffer: a #GstBuffer + * @buffer_id: a #guint64 with a unique identifier for this buffer + * @feedback: a #GstTxFeedback object implementing the #GstTxFeedbackInterface + * + * Returns: (transfer none): a #GstTxFeedbackMeta connected to @buffer + * + * Since: 1.22 + */ +GstTxFeedbackMeta * +gst_buffer_add_tx_feedback_meta (GstBuffer * buffer, + guint64 buffer_id, GstTxFeedback * feedback) +{ + GstTxFeedbackMeta *meta; + + g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL); + g_return_val_if_fail (GST_IS_TX_FEEDBACK (feedback), NULL); + + meta = (GstTxFeedbackMeta *) gst_buffer_add_meta (buffer, + GST_TX_FEEDBACK_META_INFO, NULL); + + meta->buffer_id = buffer_id; + meta->feedback = g_object_ref (feedback); + + return meta; +} + +/** + * gst_buffer_get_tx_feedback_meta: + * @buffer: a #GstBuffer + * + * Find the #GstTxFeedbackMeta on @buffer. + * + * Returns: (transfer none): the #GstTxFeedbackMeta or %NULL when there + * is no such metadata on @buffer. + * + * Since: 1.22 + */ +GstTxFeedbackMeta * +gst_buffer_get_tx_feedback_meta (GstBuffer * buffer) +{ + return (GstTxFeedbackMeta *) + gst_buffer_get_meta (buffer, GST_TX_FEEDBACK_META_API_TYPE); +} diff --git a/subprojects/gstreamer/libs/gst/net/gsttxfeedback.h b/subprojects/gstreamer/libs/gst/net/gsttxfeedback.h new file mode 100644 index 00000000000..e510ebdefb6 --- /dev/null +++ b/subprojects/gstreamer/libs/gst/net/gsttxfeedback.h @@ -0,0 +1,101 @@ +/* GStreamer + * Copyright (C) <2020> Havard Graff + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_TX_FEEDBACK_H__ +#define __GST_TX_FEEDBACK_H__ + +#include +#include + +G_BEGIN_DECLS + +typedef struct _GstTxFeedback GstTxFeedback; +typedef struct _GstTxFeedbackInterface GstTxFeedbackInterface; +typedef struct _GstTxFeedbackMeta GstTxFeedbackMeta; + +/** + * GstTxFeedbackInterface: + * @iface: the parent interface + * @tx_feedback: a transmit feedback + * + * The #GstTxFeedbackInterface interface. + * + * Since: 1.22 + */ +struct _GstTxFeedbackInterface { + GTypeInterface iface; + + /* virtual functions */ + void (*tx_feedback) (GstTxFeedback * feedback, + guint64 buffer_id, GstClockTime ts); +}; + +/** + * GstTxFeedbackMeta: + * @meta: the parent type + * @buffer_id: A #guint64 with an identifier for the current buffer + * + * Buffer metadata for transmission-time feedback. + * + * Since: 1.22 + */ +struct _GstTxFeedbackMeta { + GstMeta meta; + + guint64 buffer_id; + + /*< private >*/ + GstTxFeedback *feedback; +}; + +GST_NET_API +GType gst_tx_feedback_get_type (void); +#define GST_TYPE_TX_FEEDBACK (gst_tx_feedback_get_type ()) +#define GST_TX_FEEDBACK(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_TX_FEEDBACK, GstTxFeedback)) +#define GST_TX_FEEDBACK_GET_INTERFACE(obj) \ + (G_TYPE_INSTANCE_GET_INTERFACE ((obj), GST_TYPE_TX_FEEDBACK, GstTxFeedbackInterface)) +#define GST_IS_TX_FEEDBACK(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_TX_FEEDBACK)) +#define GST_TX_FEEDBACK_CAST(obj) ((GstTxFeedback *)(obj)) + +GST_NET_API +GType gst_tx_feedback_meta_api_get_type (void); +#define GST_TX_FEEDBACK_META_API_TYPE (gst_tx_feedback_meta_api_get_type()) + +GST_NET_API +const GstMetaInfo * gst_tx_feedback_meta_get_info (void); +#define GST_TX_FEEDBACK_META_INFO (gst_tx_feedback_meta_get_info()) + + +GST_NET_API +GstTxFeedbackMeta * gst_buffer_add_tx_feedback_meta (GstBuffer *buffer, + guint64 buffer_id, + GstTxFeedback *feedback); + +GST_NET_API +GstTxFeedbackMeta * gst_buffer_get_tx_feedback_meta (GstBuffer *buffer); + +GST_NET_API +void gst_tx_feedback_meta_set_tx_time (GstTxFeedbackMeta *meta, + GstClockTime ts); + +G_END_DECLS + +#endif /* __GST_TX_FEEDBACK_H__ */ diff --git a/subprojects/gstreamer/libs/gst/net/meson.build b/subprojects/gstreamer/libs/gst/net/meson.build index 17c63c8e1a4..99bac59f189 100644 --- a/subprojects/gstreamer/libs/gst/net/meson.build +++ b/subprojects/gstreamer/libs/gst/net/meson.build @@ -7,6 +7,7 @@ gst_net_sources = files( 'gstptpclock.c', 'gstntppacket.c', 'gstnetutils.c', + 'gsttxfeedback.c', ) gst_net_headers = files( @@ -18,6 +19,7 @@ gst_net_headers = files( 'gstnettimeprovider.h', 'gstnetutils.h', 'gstptpclock.h', + 'gsttxfeedback.h', 'net-prelude.h', 'net.h', ) diff --git a/subprojects/gstreamer/libs/gst/net/net.h b/subprojects/gstreamer/libs/gst/net/net.h index 4a11a94f239..6f5afd70b02 100644 --- a/subprojects/gstreamer/libs/gst/net/net.h +++ b/subprojects/gstreamer/libs/gst/net/net.h @@ -32,5 +32,6 @@ #include #include #include +#include #endif /* __GST_NET__H__ */ diff --git a/subprojects/gstreamer/tests/check/libs/gsttxfeedback.c b/subprojects/gstreamer/tests/check/libs/gsttxfeedback.c new file mode 100644 index 00000000000..39f184c8ddf --- /dev/null +++ b/subprojects/gstreamer/tests/check/libs/gsttxfeedback.c @@ -0,0 +1,116 @@ +/* GStreamer + * Copyright (C) <2020> Havard Graff + * + * gsttxfeedback.c: Unit test for the TX Feedback + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +typedef struct +{ + GObject object; + guint64 buffer_id; + GstClockTime ts; +} GDummyObject; + +typedef GObjectClass GDummyObjectClass; + +static void +g_dummy_object_tx_feedback (GstTxFeedback * parent, guint64 buffer_id, + GstClockTime ts) +{ + GDummyObject *obj = (GDummyObject *) parent; + obj->buffer_id = buffer_id; + obj->ts = ts; +} + +static void +g_dummy_object_tx_feedback_init (gpointer g_iface, + G_GNUC_UNUSED gpointer iface_data) +{ + GstTxFeedbackInterface *iface = g_iface; + iface->tx_feedback = g_dummy_object_tx_feedback; +} + +GType g_dummy_object_get_type (void); +G_DEFINE_TYPE_WITH_CODE (GDummyObject, g_dummy_object, + G_TYPE_OBJECT, G_IMPLEMENT_INTERFACE (GST_TYPE_TX_FEEDBACK, + g_dummy_object_tx_feedback_init)); + +#define G_TYPE_DUMMY_OBJECT g_dummy_object_get_type() + +static void +g_dummy_object_class_init (G_GNUC_UNUSED GDummyObjectClass * klass) +{ +} + +static void +g_dummy_object_init (G_GNUC_UNUSED GDummyObject * enc) +{ +} + +GST_START_TEST (test_basic) +{ + GstTxFeedbackMeta *meta; + GDummyObject *obj; + GstBuffer *buf; + guint64 buffer_id = 42; + GstClockTime ts = 123456789; + + obj = g_object_new (G_TYPE_DUMMY_OBJECT, NULL); + fail_unless (obj); + + buf = gst_buffer_new (); + + /* add TxFeedback buffer meta, with a unique ID and the receving object */ + gst_buffer_add_tx_feedback_meta (buf, buffer_id, GST_TX_FEEDBACK (obj)); + + /* verify the meta is on the buffer */ + meta = gst_buffer_get_tx_feedback_meta (buf); + fail_unless (meta); + + /* now set the transmit time of this buffer */ + gst_tx_feedback_meta_set_tx_time (meta, ts); + + /* and verify that the object was notified of this buffer being sent */ + fail_unless_equals_int (obj->buffer_id, buffer_id); + fail_unless_equals_int (obj->ts, ts); + + gst_buffer_unref (buf); + g_object_unref (obj); +} + +GST_END_TEST; + +static Suite * +gst_tx_feedback_suite (void) +{ + Suite *s = suite_create ("GstTxFeedback"); + TCase *tc_chain = tcase_create ("generic tests"); + + suite_add_tcase (s, tc_chain); + tcase_add_test (tc_chain, test_basic); + + return s; +} + +GST_CHECK_MAIN (gst_tx_feedback); diff --git a/subprojects/gstreamer/tests/check/meson.build b/subprojects/gstreamer/tests/check/meson.build index 65402791aac..4e303df5358 100644 --- a/subprojects/gstreamer/tests/check/meson.build +++ b/subprojects/gstreamer/tests/check/meson.build @@ -71,6 +71,7 @@ core_tests = [ [ 'libs/gstnetclientclock.c' ], [ 'libs/gstnettimeprovider.c' ], [ 'libs/gsttestclock.c' ], + [ 'libs/gsttxfeedback.c' ], [ 'libs/libsabi.c' ], [ 'libs/sparsefile.c' ], [ 'libs/transform1.c' ], From 98b255b2b27f577153731f0e6a76f066a065cdc9 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 12 Oct 2021 13:46:55 +0200 Subject: [PATCH 003/339] rtptwcc: recovery percentage, reordering and stats fixes rtptwcc: add recovery percentage When doing downspeeding, there is a problem trying to differentiate between "static packetloss" and "congestion" In the "static packetloss" case, you are dropping random packets at some percentage, and recovery mechanisms like RTX and FEC will work quite well, at the expense of some additional bitrate. In the "congestion" case however, you are dropping packets due to saturation of the network, and adding recovery mechanisms will actually just make the situation even worse, as you are trying to push more data on a channel that is already full. However, telling these two apart can be quite tricky, because the otherwise excellent TWCC stat of "received bitrate" will behave in a similar fashion in both cases, where the receiver is reporting less bits received than you sent. This is where recovery percentage shines. By keeping track of which recovery packet protects which media packet, and then using TWCC to get the acknowledgment that the recovery packets actually made it across, you can now disregard the lost media-packet as lost, since you know that a recovery packet for that one made it across. This is counted as a percentage, and it turns out that in the "static packetloss" case, this percentage is usually very high (towards 100%) but in the congestion case it is very low (around 0%). Hence you have an excellent tool to tell the two types of network problem apart. The immediate use for this is that when dealing with "static packetloss", downspeeding the bitrate does not actually help that much (less bits means less quality), and you can keep a high quality stream going, using your recovery mechanisms to make it near perfect, whereas in the congested case you know to quickly back off any recovery mechanisms, and downspeed your media to a more appropriate bandwidth. We cannot *only* depend on the feedback message to prune sent packets, since a miss functional receiver (those that doesn't generate a feedback on a proper rate) can potentially cause us to overflow the history of sent packets. The maximum is defined by the number of packets (using 16 bits) a twcc feedback message can contain status for. Basically being able to jump "back" and report older seqnums, as well as reporting the same packet multiple times. * introduce a configurable window in which to compute various stats * bugfixes in the twcc report generating code * optimizations around using less lists of packets * much improved handling of reordered packets Comparing with wireshark, this gives the correct size per packet on the wire vs. what is used internally. Also add the original RTP sequence-number to the logging for easy correlation. --- .../gst/rtpmanager/rtpsession.c | 161 ++- .../gst/rtpmanager/rtpsession.h | 5 +- .../gst/rtpmanager/rtpstats.c | 230 ---- .../gst/rtpmanager/rtpstats.h | 29 +- .../gst-plugins-good/gst/rtpmanager/rtptwcc.c | 1132 ++++++++++++++--- .../gst-plugins-good/gst/rtpmanager/rtptwcc.h | 42 +- .../tests/check/elements/rtpsession.c | 1037 +++++++++++++-- 7 files changed, 2046 insertions(+), 590 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 4fcd57222c5..6a02b40d0b5 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -39,6 +39,7 @@ GST_DEBUG_CATEGORY (rtp_session_debug); enum { SIGNAL_GET_SOURCE_BY_SSRC, + SIGNAL_GET_TWCC_WINDOWED_STATS, SIGNAL_ON_NEW_SSRC, SIGNAL_ON_SSRC_COLLISION, SIGNAL_ON_SSRC_VALIDATED, @@ -112,6 +113,7 @@ enum PROP_TWCC_FEEDBACK_INTERVAL, PROP_UPDATE_NTP64_HEADER_EXT, PROP_TIMEOUT_INACTIVE_SOURCES, + PROP_RTX_SSRC_MAP, PROP_LAST, }; @@ -133,6 +135,9 @@ static void rtp_session_set_property (GObject * object, guint prop_id, static void rtp_session_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static GstStructure *rtp_session_get_twcc_windowed_stats (RTPSession * sess, + GstClockTime stats_window_size, GstClockTime stats_window_delay); + static gboolean rtp_session_send_rtcp (RTPSession * sess, GstClockTime max_delay); static gboolean rtp_session_send_rtcp_with_deadline (RTPSession * sess, @@ -186,6 +191,33 @@ rtp_session_class_init (RTPSessionClass * klass) get_source_by_ssrc), NULL, NULL, NULL, RTP_TYPE_SOURCE, 1, G_TYPE_UINT); + /** + * RTPSession::get_twcc_windowed_stats: + * @session: the object which received the signal + * @stats_window_size: The size (in nanoseconds) of the stats window + * @stats_window_delay: The delay (in nanoseconds) from the current time + * until the start of the stats window. + * + * While the size of the stats window is the time to consider stats over, + * the delay is in effect a "jitterbuffer latency" for the send-side, allowing + * effects such as reordering and recovery to affect the stats, potentially + * reporting a much prettier picture of the network conditions. + * + * As an example, if the current time was 50ms, + * this would be a window size of 20ms, and a window + * delay of 10ms: + * + * timeline (ms): 0 10 20 30 40 50 + * window: [------------] + * + * Since: 1.24 + */ + rtp_session_signals[SIGNAL_GET_TWCC_WINDOWED_STATS] = + g_signal_new ("get-twcc-windowed-stats", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (RTPSessionClass, + get_twcc_windowed_stats), NULL, NULL, NULL, + GST_TYPE_STRUCTURE, 2, GST_TYPE_CLOCK_TIME, GST_TYPE_CLOCK_TIME); + /** * RTPSession::on-new-ssrc: * @session: the object which received the signal @@ -677,8 +709,22 @@ rtp_session_class_init (RTPSessionClass * klass) DEFAULT_TIMEOUT_INACTIVE_SOURCES, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + /** + * RTPSession:rtx-ssrc-map: + * + * Mapping from SSRC to RTX ssrcs. + * + * Since: 1.24 + */ + properties[PROP_RTX_SSRC_MAP] = + g_param_spec_boxed ("rtx-ssrc-map", "RTX SSRC Map", + "Map of SSRCs to their retransmission SSRCs", + GST_TYPE_STRUCTURE, G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, PROP_LAST, properties); + klass->get_twcc_windowed_stats = + GST_DEBUG_FUNCPTR (rtp_session_get_twcc_windowed_stats); klass->get_source_by_ssrc = GST_DEBUG_FUNCPTR (rtp_session_get_source_by_ssrc); klass->send_rtcp = GST_DEBUG_FUNCPTR (rtp_session_send_rtcp); @@ -769,7 +815,7 @@ rtp_session_init (RTPSession * sess) sess->is_doing_ptp = TRUE; sess->twcc = rtp_twcc_manager_new (sess->mtu); - sess->twcc_stats = rtp_twcc_stats_new (); + sess->rtx_ssrc_to_ssrc = g_hash_table_new (NULL, NULL); } static void @@ -792,7 +838,9 @@ rtp_session_finalize (GObject * object) g_hash_table_destroy (sess->ssrcs[i]); g_object_unref (sess->twcc); - rtp_twcc_stats_free (sess->twcc_stats); + if (sess->rtx_ssrc_map) + gst_structure_free (sess->rtx_ssrc_map); + g_hash_table_destroy (sess->rtx_ssrc_to_ssrc); g_mutex_clear (&sess->lock); @@ -870,6 +918,23 @@ rtp_session_create_stats (RTPSession * sess) return s; } +static gboolean +structure_to_hash_table_reverse (GQuark field_id, const GValue * value, + gpointer hash) +{ + const gchar *field_str; + guint field_uint; + guint value_uint; + + field_str = g_quark_to_string (field_id); + field_uint = atoi (field_str); + value_uint = g_value_get_uint (value); + g_hash_table_insert ((GHashTable *) hash, GUINT_TO_POINTER (value_uint), + GUINT_TO_POINTER (field_uint)); + + return TRUE; +} + static void rtp_session_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) @@ -972,6 +1037,17 @@ rtp_session_set_property (GObject * object, guint prop_id, case PROP_TIMEOUT_INACTIVE_SOURCES: sess->timeout_inactive_sources = g_value_get_boolean (value); break; + case PROP_RTX_SSRC_MAP: + RTP_SESSION_LOCK (sess); + if (sess->rtx_ssrc_map) + gst_structure_free (sess->rtx_ssrc_map); + sess->rtx_ssrc_map = g_value_dup_boxed (value); + g_hash_table_remove_all (sess->rtx_ssrc_to_ssrc); + gst_structure_foreach (sess->rtx_ssrc_map, + structure_to_hash_table_reverse, sess->rtx_ssrc_to_ssrc); + RTP_SESSION_UNLOCK (sess); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1247,6 +1323,29 @@ rtp_session_reset (RTPSession * sess) RTP_SESSION_UNLOCK (sess); } +/* must be called with RTP_SESSION_LOCK held */ +static GstCaps * +rtp_session_get_caps_unlocked (RTPSession * sess, guint8 pt) +{ + GstCaps *ret = NULL; + + RTP_SESSION_UNLOCK (sess); + + if (sess->callbacks.caps) + ret = sess->callbacks.caps (sess, pt, sess->caps_user_data); + + RTP_SESSION_LOCK (sess); + + return ret; +} + +static GstCaps * +rtp_session_twcc_caps_cb (guint8 payload, gpointer user_data) +{ + RTPSession *sess = user_data; + return rtp_session_get_caps_unlocked (sess, payload); +} + /** * rtp_session_set_callbacks: * @sess: an #RTPSession @@ -1280,6 +1379,7 @@ rtp_session_set_callbacks (RTPSession * sess, RTPSessionCallbacks * callbacks, if (callbacks->caps) { sess->callbacks.caps = callbacks->caps; sess->caps_user_data = user_data; + rtp_twcc_manager_set_callback (sess->twcc, rtp_session_twcc_caps_cb, sess); } if (callbacks->reconsider) { sess->callbacks.reconsider = callbacks->reconsider; @@ -1607,12 +1707,7 @@ source_caps (RTPSource * source, guint8 pt, RTPSession * session) { GstCaps *result = NULL; - RTP_SESSION_UNLOCK (session); - - if (session->callbacks.caps) - result = session->callbacks.caps (session, pt, session->caps_user_data); - - RTP_SESSION_LOCK (session); + result = rtp_session_get_caps_unlocked (session, pt); GST_DEBUG ("got caps %" GST_PTR_FORMAT " for pt %d", result, pt); @@ -2119,6 +2214,22 @@ rtp_session_get_source_by_ssrc (RTPSession * sess, guint32 ssrc) return result; } +static GstStructure * +rtp_session_get_twcc_windowed_stats (RTPSession * sess, + GstClockTime stats_window_size, GstClockTime stats_window_delay) +{ + GstStructure *ret; + + g_return_val_if_fail (RTP_IS_SESSION (sess), NULL); + + RTP_SESSION_LOCK (sess); + ret = rtp_twcc_manager_get_windowed_stats (sess->twcc, + stats_window_size, stats_window_delay); + RTP_SESSION_UNLOCK (sess); + + return ret; +} + /* should be called with the SESSION lock */ static guint32 rtp_session_create_new_ssrc (RTPSession * sess) @@ -2168,6 +2279,14 @@ update_packet (GstBuffer ** buffer, guint idx, RTPPacketInfo * pinfo) /* RTP header extensions */ pinfo->header_ext = gst_rtp_buffer_get_extension_bytes (&rtp, &pinfo->header_ext_bit_pattern); + + /* if RTX, store the original seqnum (OSN) and SSRC */ + if (GST_BUFFER_FLAG_IS_SET (*buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION)) { + guint8 *payload = gst_rtp_buffer_get_payload (&rtp); + if (payload) { + pinfo->rtx_osn = GST_READ_UINT16_BE (payload); + } + } } if (pinfo->ntp64_ext_id != 0 && pinfo->send && !pinfo->have_ntp64_ext) { @@ -2238,6 +2357,8 @@ update_packet_info (RTPSession * sess, RTPPacketInfo * pinfo, pinfo->marker = FALSE; pinfo->ntp64_ext_id = send ? sess->send_ntp64_ext_id : 0; pinfo->have_ntp64_ext = FALSE; + pinfo->rtx_osn = -1; + pinfo->rtx_ssrc = 0; if (is_list) { GstBufferList *list = GST_BUFFER_LIST_CAST (data); @@ -2251,6 +2372,11 @@ update_packet_info (RTPSession * sess, RTPPacketInfo * pinfo, pinfo->arrival_time = GST_BUFFER_DTS (buffer); } + if (pinfo->rtx_osn != -1) + pinfo->rtx_ssrc = + GPOINTER_TO_UINT (g_hash_table_lookup (sess->rtx_ssrc_to_ssrc, + GUINT_TO_POINTER (pinfo->ssrc))); + return res; } @@ -2978,26 +3104,23 @@ rtp_session_process_sr_req (RTPSession * sess, guint32 sender_ssrc, static void rtp_session_process_twcc (RTPSession * sess, guint32 sender_ssrc, - guint32 media_ssrc, guint8 * fci_data, guint fci_length) + guint32 media_ssrc, guint8 * fci_data, guint fci_length, + GstClockTime current_time) { - GArray *twcc_packets; GstStructure *twcc_packets_s; GstStructure *twcc_stats_s; - twcc_packets = rtp_twcc_manager_parse_fci (sess->twcc, - fci_data, fci_length * sizeof (guint32)); - if (twcc_packets == NULL) + twcc_packets_s = rtp_twcc_manager_parse_fci (sess->twcc, + fci_data, fci_length * sizeof (guint32), current_time); + if (twcc_packets_s == NULL) return; - twcc_packets_s = rtp_twcc_stats_get_packets_structure (twcc_packets); - twcc_stats_s = - rtp_twcc_stats_process_packets (sess->twcc_stats, twcc_packets); + twcc_stats_s = rtp_twcc_manager_get_windowed_stats (sess->twcc, + 300 * GST_MSECOND, 200 * GST_MSECOND); GST_DEBUG_OBJECT (sess, "Parsed TWCC: %" GST_PTR_FORMAT, twcc_packets_s); GST_INFO_OBJECT (sess, "Current TWCC stats %" GST_PTR_FORMAT, twcc_stats_s); - g_array_unref (twcc_packets); - RTP_SESSION_UNLOCK (sess); if (sess->callbacks.notify_twcc) sess->callbacks.notify_twcc (sess, twcc_packets_s, twcc_stats_s, @@ -3101,7 +3224,7 @@ rtp_session_process_feedback (RTPSession * sess, GstRTCPPacket * packet, break; case GST_RTCP_RTPFB_TYPE_TWCC: rtp_session_process_twcc (sess, sender_ssrc, media_ssrc, - fci_data, fci_length); + fci_data, fci_length, current_time); break; default: break; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h index 4afa12cd344..aef60260f9a 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h @@ -320,7 +320,8 @@ struct _RTPSession { /* Transport-wide cc-extension */ RTPTWCCManager *twcc; - RTPTWCCStats *twcc_stats; + GstStructure *rtx_ssrc_map; + GHashTable *rtx_ssrc_to_ssrc; }; /** @@ -334,6 +335,8 @@ struct _RTPSessionClass { GObjectClass parent_class; /* action signals */ + GstStructure* (*get_twcc_windowed_stats) (RTPSession *sess, + GstClockTime stats_window_size, GstClockTime stats_window_delay); RTPSource* (*get_source_by_ssrc) (RTPSession *sess, guint32 ssrc); /* signals */ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c index 0f35046f1e8..6c4662663e2 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c @@ -448,233 +448,3 @@ __g_socket_address_to_string (GSocketAddress * addr) return ret; } - -static void -_append_structure_to_value_array (GValueArray * array, GstStructure * s) -{ - GValue *val; - g_value_array_append (array, NULL); - val = g_value_array_get_nth (array, array->n_values - 1); - g_value_init (val, GST_TYPE_STRUCTURE); - g_value_take_boxed (val, s); -} - -static void -_structure_take_value_array (GstStructure * s, - const gchar * field_name, GValueArray * array) -{ - GValue value = G_VALUE_INIT; - g_value_init (&value, G_TYPE_VALUE_ARRAY); - g_value_take_boxed (&value, array); - gst_structure_take_value (s, field_name, &value); - g_value_unset (&value); -} - -GstStructure * -rtp_twcc_stats_get_packets_structure (GArray * twcc_packets) -{ - GstStructure *ret = gst_structure_new_empty ("RTPTWCCPackets"); - GValueArray *array = g_value_array_new (0); - guint i; - - for (i = 0; i < twcc_packets->len; i++) { - RTPTWCCPacket *pkt = &g_array_index (twcc_packets, RTPTWCCPacket, i); - - GstStructure *pkt_s = gst_structure_new ("RTPTWCCPacket", - "seqnum", G_TYPE_UINT, pkt->seqnum, - "local-ts", G_TYPE_UINT64, pkt->local_ts, - "remote-ts", G_TYPE_UINT64, pkt->remote_ts, - "payload-type", G_TYPE_UCHAR, pkt->pt, - "size", G_TYPE_UINT, pkt->size, - "lost", G_TYPE_BOOLEAN, pkt->status == RTP_TWCC_PACKET_STATUS_NOT_RECV, - NULL); - _append_structure_to_value_array (array, pkt_s); - } - - _structure_take_value_array (ret, "packets", array); - return ret; -} - -static void -rtp_twcc_stats_calculate_stats (RTPTWCCStats * stats, GArray * twcc_packets) -{ - guint packets_recv = 0; - guint i; - - for (i = 0; i < twcc_packets->len; i++) { - RTPTWCCPacket *pkt = &g_array_index (twcc_packets, RTPTWCCPacket, i); - - if (pkt->status != RTP_TWCC_PACKET_STATUS_NOT_RECV) - packets_recv++; - - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts) && - GST_CLOCK_TIME_IS_VALID (stats->last_local_ts)) { - pkt->local_delta = GST_CLOCK_DIFF (stats->last_local_ts, pkt->local_ts); - } - - if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) && - GST_CLOCK_TIME_IS_VALID (stats->last_remote_ts)) { - pkt->remote_delta = - GST_CLOCK_DIFF (stats->last_remote_ts, pkt->remote_ts); - } - - if (GST_CLOCK_STIME_IS_VALID (pkt->local_delta) && - GST_CLOCK_STIME_IS_VALID (pkt->remote_delta)) { - pkt->delta_delta = pkt->remote_delta - pkt->local_delta; - } - - stats->last_local_ts = pkt->local_ts; - stats->last_remote_ts = pkt->remote_ts; - } - - stats->packets_sent = twcc_packets->len; - stats->packets_recv = packets_recv; -} - -static gint -_get_window_start_index (RTPTWCCStats * stats, GstClockTime duration, - GstClockTime * local_duration, GstClockTime * remote_duration) -{ - RTPTWCCPacket *last = NULL; - guint i; - - if (stats->packets->len < 2) - return -1; - - for (i = 0; i < stats->packets->len; i++) { - guint start_index = stats->packets->len - 1 - i; - RTPTWCCPacket *pkt = - &g_array_index (stats->packets, RTPTWCCPacket, start_index); - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts) - && GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { - /* first find the last valid packet */ - if (last == NULL) { - last = pkt; - } else { - /* and then get the duration in local ts */ - GstClockTimeDiff ld = GST_CLOCK_DIFF (pkt->local_ts, last->local_ts); - if (ld >= duration) { - *local_duration = ld; - *remote_duration = GST_CLOCK_DIFF (pkt->remote_ts, last->remote_ts); - return start_index; - } - } - } - } - - return -1; -} - -static void -rtp_twcc_stats_calculate_windowed_stats (RTPTWCCStats * stats) -{ - guint i; - gint start_idx; - guint bits_sent = 0; - guint bits_recv = 0; - guint packets_sent = 0; - guint packets_recv = 0; - guint packets_lost; - GstClockTimeDiff delta_delta_sum = 0; - guint delta_delta_count = 0; - GstClockTime local_duration; - GstClockTime remote_duration; - - start_idx = _get_window_start_index (stats, stats->window_size, - &local_duration, &remote_duration); - if (start_idx == -1) { - return; - } - - /* remove the old packets */ - if (start_idx > 0) - g_array_remove_range (stats->packets, 0, start_idx); - - packets_sent = stats->packets->len - 1; - - for (i = 0; i < packets_sent; i++) { - RTPTWCCPacket *pkt = &g_array_index (stats->packets, RTPTWCCPacket, i); - - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { - bits_sent += pkt->size * 8; - } - - if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { - bits_recv += pkt->size * 8; - packets_recv++; - } - - if (GST_CLOCK_STIME_IS_VALID (pkt->delta_delta)) { - delta_delta_sum += pkt->delta_delta; - delta_delta_count++; - } - } - - packets_lost = packets_sent - packets_recv; - stats->packet_loss_pct = (packets_lost * 100) / (gfloat) packets_sent; - - if (delta_delta_count) { - GstClockTimeDiff avg_delta_of_delta = delta_delta_sum / delta_delta_count; - if (GST_CLOCK_STIME_IS_VALID (stats->avg_delta_of_delta)) { - stats->avg_delta_of_delta_change = - (avg_delta_of_delta - - stats->avg_delta_of_delta) / (250 * GST_USECOND); - } - stats->avg_delta_of_delta = avg_delta_of_delta; - } - - if (local_duration > 0) - stats->bitrate_sent = - gst_util_uint64_scale (bits_sent, GST_SECOND, local_duration); - if (remote_duration > 0) - stats->bitrate_recv = - gst_util_uint64_scale (bits_recv, GST_SECOND, remote_duration); - - GST_DEBUG ("Got stats: bits_sent: %u, bits_recv: %u, packets_sent = %u, " - "packets_recv: %u, packetlost_pct = %f, sent_bitrate = %u, " - "recv_bitrate = %u, delta-delta-avg = %" GST_STIME_FORMAT ", " - "delta-delta-change: %f", bits_sent, bits_recv, stats->packets_sent, - packets_recv, stats->packet_loss_pct, stats->bitrate_sent, - stats->bitrate_recv, GST_STIME_ARGS (stats->avg_delta_of_delta), - stats->avg_delta_of_delta_change); -} - -RTPTWCCStats * -rtp_twcc_stats_new (void) -{ - RTPTWCCStats *stats = g_new0 (RTPTWCCStats, 1); - stats->packets = g_array_new (FALSE, FALSE, sizeof (RTPTWCCPacket)); - stats->last_local_ts = GST_CLOCK_TIME_NONE; - stats->last_remote_ts = GST_CLOCK_TIME_NONE; - stats->avg_delta_of_delta = GST_CLOCK_STIME_NONE; - stats->window_size = 300 * GST_MSECOND; /* FIXME: could be configurable? */ - return stats; -} - -void -rtp_twcc_stats_free (RTPTWCCStats * stats) -{ - g_array_unref (stats->packets); - g_free (stats); -} - -static GstStructure * -rtp_twcc_stats_get_stats_structure (RTPTWCCStats * stats) -{ - return gst_structure_new ("RTPTWCCStats", - "bitrate-sent", G_TYPE_UINT, stats->bitrate_sent, - "bitrate-recv", G_TYPE_UINT, stats->bitrate_recv, - "packets-sent", G_TYPE_UINT, stats->packets_sent, - "packets-recv", G_TYPE_UINT, stats->packets_recv, - "packet-loss-pct", G_TYPE_DOUBLE, stats->packet_loss_pct, - "avg-delta-of-delta", G_TYPE_INT64, stats->avg_delta_of_delta, NULL); -} - -GstStructure * -rtp_twcc_stats_process_packets (RTPTWCCStats * stats, GArray * twcc_packets) -{ - rtp_twcc_stats_calculate_stats (stats, twcc_packets); - g_array_append_vals (stats->packets, twcc_packets->data, twcc_packets->len); - rtp_twcc_stats_calculate_windowed_stats (stats); - return rtp_twcc_stats_get_stats_structure (stats); -} diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h index 45ad377ee45..8f326273926 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h @@ -114,6 +114,8 @@ typedef struct { guint16 header_ext_bit_pattern; guint8 ntp64_ext_id; gboolean have_ntp64_ext; + gint32 rtx_osn; + guint32 rtx_ssrc; } RTPPacketInfo; /** @@ -260,27 +262,6 @@ typedef struct { guint nacks_received; } RTPSessionStats; -/** - * RTPTWCCStats: - * - * Stats kept for a session and used to produce TWCC stats. - */ -typedef struct { - GArray *packets; - GstClockTime window_size; - GstClockTime last_local_ts; - GstClockTime last_remote_ts; - - guint bitrate_sent; - guint bitrate_recv; - guint packets_sent; - guint packets_recv; - gfloat packet_loss_pct; - GstClockTimeDiff avg_delta_of_delta; - gfloat avg_delta_of_delta_change; -} RTPTWCCStats; - - void rtp_stats_init_defaults (RTPSessionStats *stats); void rtp_stats_set_bandwidths (RTPSessionStats *stats, @@ -300,10 +281,4 @@ void rtp_stats_set_min_interval (RTPSessionStats *stats, gboolean __g_socket_address_equal (GSocketAddress *a, GSocketAddress *b); gchar * __g_socket_address_to_string (GSocketAddress * addr); -RTPTWCCStats * rtp_twcc_stats_new (void); -void rtp_twcc_stats_free (RTPTWCCStats * stats); -GstStructure * rtp_twcc_stats_process_packets (RTPTWCCStats * stats, - GArray * twcc_packets); -GstStructure * rtp_twcc_stats_get_packets_structure (GArray * twcc_packets); - #endif /* __RTP_STATS_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c index 3b702ef9384..1719cf8b99c 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c @@ -17,6 +17,8 @@ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ +#define GLIB_DISABLE_DEPRECATION_WARNINGS + #include "rtptwcc.h" #include #include @@ -25,8 +27,10 @@ #include "gstrtputils.h" -GST_DEBUG_CATEGORY_EXTERN (rtp_session_debug); -#define GST_CAT_DEFAULT rtp_session_debug +GST_DEBUG_CATEGORY (rtp_twcc_debug); +#define GST_CAT_DEFAULT rtp_twcc_debug + +#define WEIGHT(a, b, w) (((a) * (w)) + ((b) * (1.0 - (w)))) #define TWCC_EXTMAP_STR "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" @@ -37,6 +41,8 @@ GST_DEBUG_CATEGORY_EXTERN (rtp_session_debug); #define STATUS_VECTOR_MAX_CAPACITY 14 #define STATUS_VECTOR_TWO_BIT_MAX_CAPACITY 7 +#define MAX_PACKETS_PER_FEEDBACK 65536 + typedef enum { RTP_TWCC_CHUNK_TYPE_RUN_LENGTH = 0, @@ -51,6 +57,13 @@ typedef struct guint8 fb_pkt_count[1]; } RTPTWCCHeader; +typedef enum +{ + RTP_TWCC_PACKET_STATUS_NOT_RECV = 0, + RTP_TWCC_PACKET_STATUS_SMALL_DELTA = 1, + RTP_TWCC_PACKET_STATUS_LARGE_NEGATIVE_DELTA = 2, +} RTPTWCCPacketStatus; + typedef struct { GstClockTime ts; @@ -64,18 +77,525 @@ typedef struct typedef struct { - GstClockTime ts; + GstClockTime local_ts; GstClockTime socket_ts; GstClockTime remote_ts; guint16 seqnum; guint8 pt; guint size; gboolean lost; + gint32 rtx_osn; + guint32 rtx_ssrc; } SentPacket; +typedef struct +{ + RTPTWCCPacketStatus status; + guint16 seqnum; + GstClockTime remote_ts; +} ParsedPacket; + +typedef struct +{ + GstClockTime org_ts; + GstClockTime local_ts; + GstClockTime remote_ts; + guint16 seqnum; + guint size; + guint8 pt; + + GstClockTimeDiff local_delta; + GstClockTimeDiff remote_delta; + GstClockTimeDiff delta_delta; + gboolean recovered; +} StatsPacket; + +static void +stats_packet_init_from_sent_packet (StatsPacket * pkt, SentPacket * sent) +{ + pkt->org_ts = sent->local_ts; + pkt->local_ts = sent->local_ts; + pkt->remote_ts = sent->remote_ts; + pkt->seqnum = sent->seqnum; + pkt->size = sent->size; + pkt->pt = sent->pt; + pkt->recovered = FALSE; + pkt->local_delta = GST_CLOCK_STIME_NONE; + pkt->remote_delta = GST_CLOCK_STIME_NONE; + pkt->delta_delta = GST_CLOCK_STIME_NONE; + + /* if we have a socket-timestamp, use that instead */ + if (GST_CLOCK_TIME_IS_VALID (sent->socket_ts)) { + pkt->local_ts = sent->socket_ts; + } +} + +typedef struct +{ + GArray *packets; + gint64 new_packets_idx; + + /* windowed stats */ + guint packets_sent; + guint packets_recv; + guint bitrate_sent; + guint bitrate_recv; + gdouble packet_loss_pct; + gdouble recovery_pct; + gint64 avg_delta_of_delta; + gdouble delta_of_delta_growth; +} TWCCStatsCtx; + +static void +_append_structure_to_value_array (GValueArray * array, GstStructure * s) +{ + GValue *val; + g_value_array_append (array, NULL); + val = g_value_array_get_nth (array, array->n_values - 1); + g_value_init (val, GST_TYPE_STRUCTURE); + g_value_take_boxed (val, s); +} + +static void +_structure_take_value_array (GstStructure * s, + const gchar * field_name, GValueArray * array) +{ + GValue value = G_VALUE_INIT; + g_value_init (&value, G_TYPE_VALUE_ARRAY); + g_value_take_boxed (&value, array); + gst_structure_take_value (s, field_name, &value); + g_value_unset (&value); +} + +static TWCCStatsCtx * +twcc_stats_ctx_new (void) +{ + TWCCStatsCtx *ctx = g_new0 (TWCCStatsCtx, 1); + + ctx->packets = g_array_new (FALSE, FALSE, sizeof (StatsPacket)); + + return ctx; +} + +static void +twcc_stats_ctx_free (TWCCStatsCtx * ctx) +{ + g_array_unref (ctx->packets); + g_free (ctx); +} + +static GstClockTime +twcc_stats_ctx_get_last_local_ts (TWCCStatsCtx * ctx) +{ + GstClockTime ret = GST_CLOCK_TIME_NONE; + StatsPacket *pkt = NULL; + if (ctx->packets->len > 0) + pkt = &g_array_index (ctx->packets, StatsPacket, ctx->packets->len - 1); + if (pkt) + ret = pkt->local_ts; + return ret; +} + +static gboolean +_get_stats_packets_window (GArray * array, + GstClockTimeDiff start_time, GstClockTimeDiff end_time, + guint * start_idx, guint * num_packets) +{ + gboolean ret = FALSE; + guint end_idx = 0; + guint i; + + if (array->len < 2) { + GST_DEBUG ("Not enough starts to do a window"); + return FALSE; + } + + for (i = 0; i < array->len; i++) { + StatsPacket *pkt = &g_array_index (array, StatsPacket, i); + if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { + GstClockTimeDiff offset = GST_CLOCK_DIFF (pkt->local_ts, start_time); + *start_idx = i; + /* positive number here means it is older than our start time */ + if (offset > 0) { + GST_LOG ("Packet #%u is too old: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); + } else { + GST_LOG ("Setting first packet in our window to #%u: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); + ret = TRUE; + break; + } + } + } + + /* jump out early if we could not find a start_idx */ + if (!ret) { + return FALSE; + } + + ret = FALSE; + for (i = 0; i < array->len - *start_idx - 1; i++) { + guint idx = array->len - 1 - i; + StatsPacket *pkt = &g_array_index (array, StatsPacket, idx); + if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { + GstClockTimeDiff offset = GST_CLOCK_DIFF (pkt->local_ts, end_time); + if (offset >= 0) { + GST_LOG ("Setting last packet in our window to #%u: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); + end_idx = idx; + ret = TRUE; + break; + } else { + GST_LOG ("Packet #%u is too new: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); + } + } + } + + /* jump out early if we could not find a window */ + if (!ret) { + return FALSE; + } + + *num_packets = end_idx - *start_idx + 1; + + return ret; +} + +static gboolean +twcc_stats_ctx_calculate_windowed_stats (TWCCStatsCtx * ctx, + GstClockTimeDiff start_time, GstClockTimeDiff end_time) +{ + GArray *packets = ctx->packets; + guint start_idx; + guint packets_sent = 0; + guint packets_recv = 0; + guint packets_recovered = 0; + guint packets_lost = 0; + + /* FIXME: property ? */ + guint max_stats_packets = 1000; + + guint i; + guint bits_sent = 0; + guint bits_recv = 0; + + GstClockTimeDiff delta_delta_sum = 0; + guint delta_delta_count = 0; + GstClockTimeDiff first_delta_delta_sum = 0; + guint first_delta_delta_count = 0; + GstClockTimeDiff last_delta_delta_sum = 0; + guint last_delta_delta_count = 0; + + StatsPacket *first_local_pkt = NULL; + StatsPacket *last_local_pkt = NULL; + StatsPacket *first_remote_pkt = NULL; + StatsPacket *last_remote_pkt = NULL; + + GstClockTimeDiff local_duration = 0; + GstClockTimeDiff remote_duration = 0; + + ctx->packet_loss_pct = 0.0; + ctx->avg_delta_of_delta = 0; + ctx->delta_of_delta_growth = 0.0; + ctx->bitrate_sent = 0; + ctx->bitrate_recv = 0; + ctx->recovery_pct = -1.0; + + gboolean ret = + _get_stats_packets_window (packets, start_time, end_time, &start_idx, + &packets_sent); + if (!ret || packets_sent < 2) { + GST_INFO ("Not enough packets to fill our window yet!"); + return FALSE; + } + + for (i = 0; i < packets_sent; i++) { + StatsPacket *pkt = &g_array_index (packets, StatsPacket, i + start_idx); + GST_LOG ("STATS WINDOW: %u/%u: pkt #%u, pt: %u, size: %u, arrived: %s, " + "local-ts: %" GST_TIME_FORMAT ", remote-ts %" GST_TIME_FORMAT, + i + 1, packets_sent, pkt->seqnum, pkt->pt, pkt->size * 8, + GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) ? "YES" : "NO", + GST_TIME_ARGS (pkt->local_ts), GST_TIME_ARGS (pkt->remote_ts)); + + if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { + /* don't count the bits for the first packet in the window */ + if (first_local_pkt == NULL) { + first_local_pkt = pkt; + } else { + bits_sent += pkt->size * 8; + } + last_local_pkt = pkt; + } + + if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { + /* don't count the bits for the first packet in the window */ + if (first_remote_pkt == NULL) { + first_remote_pkt = pkt; + } else { + bits_recv += pkt->size * 8; + } + last_remote_pkt = pkt; + packets_recv++; + } else { + GST_LOG ("Packet #%u is lost, recovered=%u", pkt->seqnum, pkt->recovered); + packets_lost++; + if (pkt->recovered) { + packets_recovered++; + } + } + + if (GST_CLOCK_STIME_IS_VALID (pkt->delta_delta)) { + delta_delta_sum += pkt->delta_delta; + delta_delta_count++; + if (i < packets_sent / 2) { + first_delta_delta_sum += pkt->delta_delta; + first_delta_delta_count++; + } else { + last_delta_delta_sum += pkt->delta_delta; + last_delta_delta_count++; + } + } + } + + ctx->packets_sent = packets_sent; + ctx->packets_recv = packets_recv; + + if (first_local_pkt && last_local_pkt) { + local_duration = + GST_CLOCK_DIFF (first_local_pkt->local_ts, last_local_pkt->local_ts); + } + if (first_remote_pkt && last_remote_pkt) { + remote_duration = + GST_CLOCK_DIFF (first_remote_pkt->remote_ts, + last_remote_pkt->remote_ts); + } + + if (packets_sent) + ctx->packet_loss_pct = (packets_lost * 100) / (gfloat) packets_sent; + + if (packets_lost) { + ctx->recovery_pct = (packets_recovered * 100) / (gfloat) packets_lost; + } + + if (delta_delta_count) { + ctx->avg_delta_of_delta = delta_delta_sum / delta_delta_count; + } + + if (first_delta_delta_count && last_delta_delta_count) { + GstClockTimeDiff first_avg = + first_delta_delta_sum / first_delta_delta_count; + GstClockTimeDiff last_avg = last_delta_delta_sum / last_delta_delta_count; + + /* filter out very small numbers */ + first_avg = MAX (first_avg, 100 * GST_USECOND); + last_avg = MAX (last_avg, 100 * GST_USECOND); + ctx->delta_of_delta_growth = (double) last_avg / (double) first_avg; + } + + if (local_duration > 0) { + ctx->bitrate_sent = + gst_util_uint64_scale (bits_sent, GST_SECOND, local_duration); + } + if (remote_duration > 0) { + ctx->bitrate_recv = + gst_util_uint64_scale (bits_recv, GST_SECOND, remote_duration); + } + + GST_INFO ("Got stats: bits_sent: %u, bits_recv: %u, packets_sent = %u, " + "packets_recv: %u, packetlost_pct = %lf, recovery_pct = %lf, " + "local_duration=%" GST_TIME_FORMAT ", remote_duration=%" GST_TIME_FORMAT + ", " "sent_bitrate = %u, " "recv_bitrate = %u, delta-delta-avg = %" + GST_STIME_FORMAT ", delta-delta-growth=%lf", bits_sent, bits_recv, + packets_sent, packets_recv, ctx->packet_loss_pct, ctx->recovery_pct, + GST_TIME_ARGS (local_duration), GST_TIME_ARGS (remote_duration), + ctx->bitrate_sent, ctx->bitrate_recv, + GST_STIME_ARGS (ctx->avg_delta_of_delta), ctx->delta_of_delta_growth); + + /* trim the stats array down to max packets */ + if (packets->len > max_stats_packets) { + g_array_remove_range (packets, 0, packets->len - max_stats_packets); + } + + return TRUE; +} + +static GstStructure * +twcc_stats_ctx_get_structure (TWCCStatsCtx * ctx) +{ + return gst_structure_new ("RTPTWCCStats", + "packets-sent", G_TYPE_UINT, ctx->packets_sent, + "packets-recv", G_TYPE_UINT, ctx->packets_recv, + "bitrate-sent", G_TYPE_UINT, ctx->bitrate_sent, + "bitrate-recv", G_TYPE_UINT, ctx->bitrate_recv, + "packet-loss-pct", G_TYPE_DOUBLE, ctx->packet_loss_pct, + "recovery-pct", G_TYPE_DOUBLE, ctx->recovery_pct, + "avg-delta-of-delta", G_TYPE_INT64, ctx->avg_delta_of_delta, + "delta-of-delta-growth", G_TYPE_DOUBLE, ctx->delta_of_delta_growth, NULL); +} + + +static void +_update_stats_for_packet_idx (GArray * array, gint packet_idx) +{ + StatsPacket *pkt; + StatsPacket *prev; + + /* we need at least 2 packets to do any stats */ + if (array->len < 2) { + GST_DEBUG ("Not yet 2 packets in stats"); + return; + } + + /* we can't do stats on the first packet */ + if (packet_idx == 0) { + GST_DEBUG ("First packet can't have stats calculated"); + return; + } + + /* packet_idx must be inside the array */ + if (array->len <= packet_idx) { + GST_DEBUG ("Packet idx %u is outside of array!", packet_idx); + return; + } + + pkt = &g_array_index (array, StatsPacket, packet_idx); + prev = &g_array_index (array, StatsPacket, packet_idx - 1); + + if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts) && + GST_CLOCK_TIME_IS_VALID (prev->local_ts)) { + pkt->local_delta = GST_CLOCK_DIFF (prev->local_ts, pkt->local_ts); + GST_LOG ("Calculated local_delta for packet #%u to %" GST_STIME_FORMAT, + pkt->seqnum, GST_STIME_ARGS (pkt->local_delta)); + } + + if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) && + GST_CLOCK_TIME_IS_VALID (prev->remote_ts)) { + pkt->remote_delta = GST_CLOCK_DIFF (prev->remote_ts, pkt->remote_ts); + GST_LOG ("Calculated remote_delta for packet #%u to %" GST_STIME_FORMAT, + pkt->seqnum, GST_STIME_ARGS (pkt->remote_delta)); + } + + if (GST_CLOCK_STIME_IS_VALID (pkt->local_delta) && + GST_CLOCK_STIME_IS_VALID (pkt->remote_delta)) { + pkt->delta_delta = pkt->remote_delta - pkt->local_delta; + GST_LOG ("Calculated delta-of-delta for packet #%u to %" GST_STIME_FORMAT, + pkt->seqnum, GST_STIME_ARGS (pkt->delta_delta)); + } +} + +static gint +_get_packet_idx_for_seqnum (GArray * array, guint16 seqnum) +{ + guint i; + + for (i = 0; i < array->len; i++) { + StatsPacket *pkt = &g_array_index (array, StatsPacket, i); + if (pkt->seqnum == seqnum) { + return (gint) i; + } + } + + return -1; +} + +/* assumes all seqnum are in order */ +static gint +_get_packet_idx_for_seqnum_fast (GArray * array, guint16 seqnum) +{ + StatsPacket *first; + guint16 idx; + + if (array->len == 0) + return -1; + + first = &g_array_index (array, StatsPacket, 0); + idx = gst_rtp_buffer_compare_seqnum (first->seqnum, seqnum); + + if (idx < array->len) { + StatsPacket *found = &g_array_index (array, StatsPacket, idx); + if (found->seqnum == seqnum) { + return (gint) idx; + } + } + + return -1; +} + +static gint +twcc_stats_ctx_get_packet_idx_for_seqnum (TWCCStatsCtx * ctx, guint16 seqnum) +{ + gint ret; + + /* assumes all packets seqnum are in order */ + ret = _get_packet_idx_for_seqnum_fast (ctx->packets, seqnum); + if (ret != -1) + return ret; + + return _get_packet_idx_for_seqnum (ctx->packets, seqnum); +} + +static StatsPacket * +twcc_stats_ctx_get_packet_for_seqnum (TWCCStatsCtx * ctx, guint16 seqnum) +{ + StatsPacket *ret = NULL; + gint idx = twcc_stats_ctx_get_packet_idx_for_seqnum (ctx, seqnum); + if (idx != -1) + ret = &g_array_index (ctx->packets, StatsPacket, idx); + + return ret; +} + +static void +twcc_stats_ctx_add_packet (TWCCStatsCtx * ctx, StatsPacket * pkt) +{ + StatsPacket *last = NULL; + gint pkt_idx; + + if (ctx->packets->len > 0) + last = &g_array_index (ctx->packets, StatsPacket, ctx->packets->len - 1); + + /* first a quick check to see if we are going forward in seqnum, + in which case we simply append */ + if (last == NULL || pkt->seqnum > last->seqnum) { + GST_LOG ("Appending #%u to stats packets", pkt->seqnum); + g_array_append_val (ctx->packets, *pkt); + /* and update the stats for this packet */ + _update_stats_for_packet_idx (ctx->packets, ctx->packets->len - 1); + return; + } + + /* here we are dealing with a reordered packet, we start by searching for it */ + pkt_idx = twcc_stats_ctx_get_packet_idx_for_seqnum (ctx, pkt->seqnum); + if (pkt_idx != -1) { + StatsPacket *existing = &g_array_index (ctx->packets, StatsPacket, pkt_idx); + /* only update if we don't already have information about this packet, and + this packet brings something new */ + if (!GST_CLOCK_TIME_IS_VALID (existing->remote_ts) && + GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { + GST_DEBUG ("Updating stats packet #%u", pkt->seqnum); + g_array_index (ctx->packets, StatsPacket, pkt_idx) = *pkt; + + /* now update stats for this packet and the next one along */ + _update_stats_for_packet_idx (ctx->packets, pkt_idx); + _update_stats_for_packet_idx (ctx->packets, pkt_idx + 1); + } + return; + } +} + +/******************************************************/ + struct _RTPTWCCManager { GObject object; + GMutex recv_lock; + + GHashTable *ssrc_to_seqmap; + GHashTable *pt_to_twcc_ext_id; + + TWCCStatsCtx *stats_ctx; + GHashTable *stats_ctx_by_pt; guint8 send_ext_id; guint8 recv_ext_id; @@ -86,7 +606,6 @@ struct _RTPTWCCManager GArray *recv_packets; guint64 fb_pkt_count; - gint32 last_seqnum; GArray *sent_packets; GArray *parsed_packets; @@ -104,6 +623,12 @@ struct _RTPTWCCManager GstClockTime next_feedback_send_time; GstClockTime feedback_interval; + + GstClockTimeDiff avg_rtt; + GstClockTime last_report_time; + + RTPTWCCManagerCaps caps_cb; + gpointer caps_ud; }; @@ -123,13 +648,21 @@ G_DEFINE_TYPE_WITH_CODE (RTPTWCCManager, rtp_twcc_manager, G_TYPE_OBJECT, static void rtp_twcc_manager_init (RTPTWCCManager * twcc) { + twcc->ssrc_to_seqmap = g_hash_table_new_full (NULL, NULL, NULL, + (GDestroyNotify) g_hash_table_destroy); + twcc->pt_to_twcc_ext_id = g_hash_table_new (NULL, NULL); + twcc->recv_packets = g_array_new (FALSE, FALSE, sizeof (RecvPacket)); twcc->sent_packets = g_array_new (FALSE, FALSE, sizeof (SentPacket)); - twcc->parsed_packets = g_array_new (FALSE, FALSE, sizeof (RecvPacket)); + twcc->parsed_packets = g_array_new (FALSE, FALSE, sizeof (ParsedPacket)); + g_mutex_init (&twcc->recv_lock); twcc->rtcp_buffers = g_queue_new (); - twcc->last_seqnum = -1; + twcc->stats_ctx = twcc_stats_ctx_new (); + twcc->stats_ctx_by_pt = g_hash_table_new_full (NULL, NULL, + NULL, (GDestroyNotify) twcc_stats_ctx_free); + twcc->recv_media_ssrc = -1; twcc->recv_sender_ssrc = -1; @@ -137,6 +670,7 @@ rtp_twcc_manager_init (RTPTWCCManager * twcc) twcc->feedback_interval = GST_CLOCK_TIME_NONE; twcc->next_feedback_send_time = GST_CLOCK_TIME_NONE; + twcc->last_report_time = GST_CLOCK_TIME_NONE; } static void @@ -144,10 +678,17 @@ rtp_twcc_manager_finalize (GObject * object) { RTPTWCCManager *twcc = RTP_TWCC_MANAGER_CAST (object); + g_hash_table_destroy (twcc->ssrc_to_seqmap); + g_hash_table_destroy (twcc->pt_to_twcc_ext_id); + g_array_unref (twcc->recv_packets); g_array_unref (twcc->sent_packets); g_array_unref (twcc->parsed_packets); g_queue_free_full (twcc->rtcp_buffers, (GDestroyNotify) gst_buffer_unref); + g_mutex_clear (&twcc->recv_lock); + + g_hash_table_destroy (twcc->stats_ctx_by_pt); + twcc_stats_ctx_free (twcc->stats_ctx); G_OBJECT_CLASS (rtp_twcc_manager_parent_class)->finalize (object); } @@ -157,6 +698,8 @@ rtp_twcc_manager_class_init (RTPTWCCManagerClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; gobject_class->finalize = rtp_twcc_manager_finalize; + + GST_DEBUG_CATEGORY_INIT (rtp_twcc_debug, "rtptwcc", 0, "RTP TWCC Manager"); } RTPTWCCManager * @@ -169,6 +712,88 @@ rtp_twcc_manager_new (guint mtu) return twcc; } +static TWCCStatsCtx * +_get_ctx_for_pt (RTPTWCCManager * twcc, guint pt) +{ + TWCCStatsCtx *ctx = + g_hash_table_lookup (twcc->stats_ctx_by_pt, GUINT_TO_POINTER (pt)); + if (!ctx) { + ctx = twcc_stats_ctx_new (); + g_hash_table_insert (twcc->stats_ctx_by_pt, GUINT_TO_POINTER (pt), ctx); + } + return ctx; +} + + +static void +_update_stats_with_recovered (RTPTWCCManager * twcc, guint16 seqnum) +{ + TWCCStatsCtx *ctx; + StatsPacket *pkt = + twcc_stats_ctx_get_packet_for_seqnum (twcc->stats_ctx, seqnum); + + if (pkt == NULL) { + GST_INFO ("Could not find seqnum %u", seqnum); + return; + } + + pkt->recovered = TRUE; + + /* now find the equivalent packet in the payload */ + ctx = _get_ctx_for_pt (twcc, pkt->pt); + pkt = twcc_stats_ctx_get_packet_for_seqnum (ctx, seqnum); + + if (pkt) { + pkt->recovered = TRUE; + } +} + +static gint32 +_lookup_seqnum (RTPTWCCManager * twcc, guint32 ssrc, guint16 seqnum) +{ + gint32 ret = -1; + + GHashTable *seq_to_twcc = + g_hash_table_lookup (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc)); + if (seq_to_twcc) { + ret = + GPOINTER_TO_UINT (g_hash_table_lookup (seq_to_twcc, + GUINT_TO_POINTER (seqnum))); + } + return ret; +} + +static void +_add_packet_to_stats (RTPTWCCManager * twcc, + ParsedPacket * parsed_pkt, SentPacket * sent_pkt) +{ + TWCCStatsCtx *ctx; + StatsPacket pkt; + + stats_packet_init_from_sent_packet (&pkt, sent_pkt); + + /* add packet to the stats context */ + twcc_stats_ctx_add_packet (twcc->stats_ctx, &pkt); + + /* add packet to the payload specific stats context */ + ctx = _get_ctx_for_pt (twcc, pkt.pt); + twcc_stats_ctx_add_packet (ctx, &pkt); + + /* extra check to see if we can confirm the arrival of a recovery packet, + and hence set the "original" packet as recovered */ + if (parsed_pkt->status != RTP_TWCC_PACKET_STATUS_NOT_RECV + && sent_pkt->rtx_osn != -1) { + gint32 recovered_seq = + _lookup_seqnum (twcc, sent_pkt->rtx_ssrc, sent_pkt->rtx_osn); + if (recovered_seq != -1) { + GST_LOG ("RTX Packet %u protects seqnum %d", sent_pkt->seqnum, + recovered_seq); + _update_stats_with_recovered (twcc, recovered_seq); + } + } +} + + static void recv_packet_init (RecvPacket * packet, guint16 seqnum, RTPPacketInfo * pinfo) { @@ -250,12 +875,74 @@ sent_packet_init (SentPacket * packet, guint16 seqnum, RTPPacketInfo * pinfo, GstRTPBuffer * rtp) { packet->seqnum = seqnum; - packet->ts = pinfo->current_time; - packet->size = gst_rtp_buffer_get_payload_len (rtp); + packet->local_ts = pinfo->current_time; + packet->size = pinfo->bytes + 12; /* the reported wireshark size */ packet->pt = gst_rtp_buffer_get_payload_type (rtp); packet->remote_ts = GST_CLOCK_TIME_NONE; packet->socket_ts = GST_CLOCK_TIME_NONE; packet->lost = FALSE; + packet->rtx_osn = pinfo->rtx_osn; + packet->rtx_ssrc = pinfo->rtx_ssrc; +} + +static void +rtp_twcc_manager_register_seqnum (RTPTWCCManager * twcc, + guint32 ssrc, guint16 seqnum, guint16 twcc_seqnum) +{ + GHashTable *seq_to_twcc = + g_hash_table_lookup (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc)); + if (!seq_to_twcc) { + seq_to_twcc = g_hash_table_new (NULL, NULL); + g_hash_table_insert (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc), + seq_to_twcc); + } + g_hash_table_insert (seq_to_twcc, GUINT_TO_POINTER (seqnum), + GUINT_TO_POINTER (twcc_seqnum)); + GST_LOG ("Registering OSN: %u to twcc-twcc_seqnum: %u with ssrc: %u", seqnum, + twcc_seqnum, ssrc); +} + +/* Remove old sent packets and keep them under a maximum threshold, as we + can't accumulate them if we don't get a feedback message from the + receiver. */ +static void +_prune_old_sent_packets (RTPTWCCManager * twcc) +{ + guint length; + + if (twcc->sent_packets->len <= MAX_PACKETS_PER_FEEDBACK) + return; + + length = twcc->sent_packets->len - MAX_PACKETS_PER_FEEDBACK; + g_array_remove_range (twcc->sent_packets, 0, length); +} + +/* +* Returns a pointer to the twcc extension header data for the given id, or adds +* it, if it is not present. +* +* Note: we want to override the extension value if it is present. +*/ +static gpointer +_get_twcc_buffer_ext_data (GstRTPBuffer * rtpbuf, guint8 ext_id) +{ + gpointer data = NULL; + guint16 tmp = 0; + gboolean added; + + if (gst_rtp_buffer_get_extension_onebyte_header (rtpbuf, ext_id, 0, &data, + NULL)) { + return data; + } + + added = gst_rtp_buffer_add_extension_onebyte_header (rtpbuf, ext_id, &tmp, + sizeof (tmp)); + if (added) { + gst_rtp_buffer_get_extension_onebyte_header (rtpbuf, ext_id, 0, &data, + NULL); + } + + return data; } static void @@ -265,25 +952,34 @@ _set_twcc_seqnum_data (RTPTWCCManager * twcc, RTPPacketInfo * pinfo, SentPacket packet; GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; gpointer data; + guint16 seqnum; - if (gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp)) { - if (gst_rtp_buffer_get_extension_onebyte_header (&rtp, - ext_id, 0, &data, NULL)) { - guint16 seqnum = twcc->send_seqnum++; - - GST_WRITE_UINT16_BE (data, seqnum); - sent_packet_init (&packet, seqnum, pinfo, &rtp); - g_array_append_val (twcc->sent_packets, packet); - - gst_buffer_add_tx_feedback_meta (pinfo->data, seqnum, - GST_TX_FEEDBACK_CAST (twcc)); + if (!gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp)) { + GST_WARNING ("Couldn't map the buffer %" GST_PTR_FORMAT, buf); + return; + } - GST_LOG ("Send: twcc-seqnum: %u, pt: %u, marker: %d, len: %u, ts: %" - GST_TIME_FORMAT, seqnum, packet.pt, pinfo->marker, packet.size, - GST_TIME_ARGS (pinfo->current_time)); - } + data = _get_twcc_buffer_ext_data (&rtp, ext_id); + if (!data) { gst_rtp_buffer_unmap (&rtp); + return; } + + seqnum = twcc->send_seqnum++; + GST_WRITE_UINT16_BE (data, seqnum); + sent_packet_init (&packet, seqnum, pinfo, &rtp); + gst_rtp_buffer_unmap (&rtp); + g_array_append_val (twcc->sent_packets, packet); + _prune_old_sent_packets (twcc); + rtp_twcc_manager_register_seqnum (twcc, pinfo->ssrc, pinfo->seqnum, seqnum); + + gst_buffer_add_tx_feedback_meta (pinfo->data, seqnum, + GST_TX_FEEDBACK_CAST (twcc)); + + GST_LOG + ("Send: twcc-seqnum: %u, seqnum: %u, pt: %u, marker: %d, size: %u, ts: %" + GST_TIME_FORMAT, packet.seqnum, pinfo->seqnum, packet.pt, pinfo->marker, + packet.size, GST_TIME_ARGS (pinfo->current_time)); } static void @@ -383,7 +1079,6 @@ rtp_twcc_write_run_length_chunk (GArray * packet_chunks, guint len = MIN (run_length - written, 8191); GST_LOG ("Writing a run-length of %u with status %u", len, status); - gst_bit_writer_init_with_data (&writer, (guint8 *) & data, 2, FALSE); gst_bit_writer_put_bits_uint8 (&writer, RTP_TWCC_CHUNK_TYPE_RUN_LENGTH, 1); gst_bit_writer_put_bits_uint8 (&writer, status, 2); @@ -580,8 +1275,12 @@ rtp_twcc_write_chunks (GArray * packet_chunks, /* first write in any preceeding gaps, we use run-length if it would take up more than one chunk (14/7) */ if (pkt->missing_run > packets_per_chunks) { + GST_LOG ("Missing run of %u, using run-length chunk", pkt->missing_run); rtp_twcc_write_run_length_chunk (packet_chunks, RTP_TWCC_PACKET_STATUS_NOT_RECV, pkt->missing_run); + /* for this case we need to set the missing-run to 0, or else + the status vector chunk will use it and write its own version */ + pkt->missing_run = 0; } /* we have a run of the same status, write a run-length chunk and skip @@ -603,8 +1302,10 @@ rtp_twcc_write_chunks (GArray * packet_chunks, chunk_bit_writer_flush (&writer); } +/* must be called with recv_lock */ static void -rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) +rtp_twcc_manager_add_fci_unlocked (RTPTWCCManager * twcc, + GstRTCPPacket * packet) { RecvPacket *first, *last, *prev; guint16 packet_count; @@ -625,22 +1326,7 @@ rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) GstClockTimeDiff delta_ts; gint64 delta_ts_rounded; guint8 fb_pkt_count; - - g_array_sort (twcc->recv_packets, _twcc_seqnum_sort); - - /* Quick scan to remove duplicates */ - prev = &g_array_index (twcc->recv_packets, RecvPacket, 0); - for (i = 1; i < twcc->recv_packets->len;) { - RecvPacket *cur = &g_array_index (twcc->recv_packets, RecvPacket, i); - - if (prev->seqnum == cur->seqnum) { - GST_DEBUG ("Removing duplicate packet #%u", cur->seqnum); - g_array_remove_index (twcc->recv_packets, i); - } else { - prev = cur; - i += 1; - } - } + gboolean missing_packets = FALSE; /* get first and last packet */ first = &g_array_index (twcc->recv_packets, RecvPacket, 0); @@ -671,7 +1357,9 @@ rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) prev = first; for (i = 0; i < twcc->recv_packets->len; i++) { RecvPacket *pkt = &g_array_index (twcc->recv_packets, RecvPacket, i); - if (i != 0) { + if (i == 0) { + pkt->missing_run = 0; + } else { pkt->missing_run = pkt->seqnum - prev->seqnum - 1; } @@ -690,6 +1378,9 @@ rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) } run_lenght_helper_update (&rlh, pkt); + if (pkt->missing_run > 0) + missing_packets = TRUE; + GST_LOG ("pkt: #%u, ts: %" GST_TIME_FORMAT " ts_rounded: %" GST_TIME_FORMAT " delta_ts: %" GST_STIME_FORMAT @@ -729,11 +1420,22 @@ rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) GST_MEMDUMP ("full fci:", fci_data, fci_length); g_array_unref (packet_chunks); - g_array_set_size (twcc->recv_packets, 0); + + /* If we have any missing packets in this report, keep the last packet around, + potentially reporting it several times. + This is mimicing Chrome WebRTC behavior. */ + if (missing_packets) { + twcc->recv_packets = + g_array_remove_range (twcc->recv_packets, 0, + twcc->recv_packets->len - 1); + } else { + g_array_set_size (twcc->recv_packets, 0); + } } +/* must be called with the recv_lock */ static void -rtp_twcc_manager_create_feedback (RTPTWCCManager * twcc) +rtp_twcc_manager_create_feedback_unlocked (RTPTWCCManager * twcc) { GstBuffer *buf; GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; @@ -750,7 +1452,7 @@ rtp_twcc_manager_create_feedback (RTPTWCCManager * twcc) gst_rtcp_packet_fb_set_sender_ssrc (&packet, twcc->recv_sender_ssrc); gst_rtcp_packet_fb_set_media_ssrc (&packet, twcc->recv_media_ssrc); - rtp_twcc_manager_add_fci (twcc, &packet); + rtp_twcc_manager_add_fci_unlocked (twcc, &packet); gst_rtcp_buffer_unmap (&rtcp); @@ -784,11 +1486,6 @@ _many_packets_some_lost (RTPTWCCManager * twcc, guint16 seqnum) first = &g_array_index (twcc->recv_packets, RecvPacket, 0); packet_count = seqnum - first->seqnum + 1; - /* If there are a high number of duplicates, we can't use the following - * metrics */ - if (received_packets > packet_count) - return FALSE; - /* check if we lost half of the threshold */ lost_packets = packet_count - received_packets; if (received_packets >= 30 && lost_packets >= 60) @@ -801,6 +1498,17 @@ _many_packets_some_lost (RTPTWCCManager * twcc, guint16 seqnum) return FALSE; } +static gboolean +_find_seqnum_in_recv_packets (RTPTWCCManager * twcc, guint16 seqnum) +{ + for (guint i = 0; i < twcc->recv_packets->len; i++) { + RecvPacket *pkt = &g_array_index (twcc->recv_packets, RecvPacket, i); + if (pkt->seqnum == seqnum) + return TRUE; + } + return FALSE; +} + gboolean rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) { @@ -808,17 +1516,20 @@ rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) RecvPacket packet; gint32 seqnum; gint diff; + gboolean reordered_packet = FALSE; seqnum = rtp_twcc_manager_get_recv_twcc_seqnum (twcc, pinfo); if (seqnum == -1) return FALSE; + g_mutex_lock (&twcc->recv_lock); + /* if this packet would exceed the capacity of our MTU, we create a feedback with the current packets, and start over with this one */ if (_exceeds_max_packets (twcc, seqnum)) { GST_INFO ("twcc-seqnum: %u would overflow max packets: %u, create feedback" " with current packets", seqnum, twcc->max_packets_per_rtcp); - rtp_twcc_manager_create_feedback (twcc); + rtp_twcc_manager_create_feedback_unlocked (twcc); send_feedback = TRUE; } @@ -826,52 +1537,49 @@ rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) if (twcc->recv_media_ssrc == -1) twcc->recv_media_ssrc = pinfo->ssrc; - /* check if we are reordered, and treat it as lost if we already sent - a feedback msg with a higher seqnum. If the diff is huge, treat - it as a restart of a stream */ - diff = gst_rtp_buffer_compare_seqnum (twcc->expected_recv_seqnum, seqnum); - if (twcc->fb_pkt_count > 0 && diff < 0) { - GST_INFO ("Received out of order packet (%u after %u), treating as lost", - seqnum, twcc->expected_recv_seqnum); - return FALSE; + if (twcc->recv_packets->len > 0) { + RecvPacket *last = &g_array_index (twcc->recv_packets, RecvPacket, + twcc->recv_packets->len - 1); + diff = gst_rtp_buffer_compare_seqnum (last->seqnum, seqnum); + if (diff <= 0) { + /* duplicate check */ + if (_find_seqnum_in_recv_packets (twcc, seqnum)) { + GST_INFO ("Received duplicate packet (#%u), dropping", seqnum); + g_mutex_unlock (&twcc->recv_lock); + return FALSE; + } + /* if not duplicate, it is reordered */ + GST_INFO ("Received a reordered packet (#%u)", seqnum); + reordered_packet = TRUE; + } } /* store the packet for Transport-wide RTCP feedback message */ recv_packet_init (&packet, seqnum, pinfo); g_array_append_val (twcc->recv_packets, packet); - twcc->last_seqnum = seqnum; - GST_LOG ("Receive: twcc-seqnum: %u, pt: %u, marker: %d, ts: %" + /* if we received a reordered packet, we need to sort the list */ + if (reordered_packet) + g_array_sort (twcc->recv_packets, _twcc_seqnum_sort); + + GST_LOG ("Receive: twcc-seqnum: #%u, pt: %u, marker: %d, ts: %" GST_TIME_FORMAT, seqnum, pinfo->pt, pinfo->marker, - GST_TIME_ARGS (pinfo->arrival_time)); + GST_TIME_ARGS (packet.ts)); if (!pinfo->marker) twcc->packet_count_no_marker++; - /* are we sending on an interval, or based on marker bit */ - if (GST_CLOCK_TIME_IS_VALID (twcc->feedback_interval)) { - if (!GST_CLOCK_TIME_IS_VALID (twcc->next_feedback_send_time)) - twcc->next_feedback_send_time = - pinfo->running_time + twcc->feedback_interval; - - if (pinfo->running_time >= twcc->next_feedback_send_time) { - GST_LOG ("Generating feedback : Exceeded feedback interval %" - GST_TIME_FORMAT, GST_TIME_ARGS (twcc->feedback_interval)); - rtp_twcc_manager_create_feedback_unlocked (twcc); - send_feedback = TRUE; - - while (pinfo->running_time >= twcc->next_feedback_send_time) - twcc->next_feedback_send_time += twcc->feedback_interval; - } - } else if (pinfo->marker || _many_packets_some_lost (twcc, seqnum)) { - GST_LOG ("Generating feedback because of %s", - pinfo->marker ? "marker packet" : "many packets some lost"); + /* Create feedback, if sending based on marker bit */ + if (!GST_CLOCK_TIME_IS_VALID (twcc->feedback_interval) && + (pinfo->marker || _many_packets_some_lost (twcc, seqnum))) { rtp_twcc_manager_create_feedback_unlocked (twcc); send_feedback = TRUE; twcc->packet_count_no_marker = 0; } + g_mutex_unlock (&twcc->recv_lock); + return send_feedback; } @@ -892,7 +1600,7 @@ rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint sender_ssrc, { GstBuffer *buf; - GST_DEBUG ("considering twcc. now: %" GST_TIME_FORMAT + GST_LOG ("considering twcc. now: %" GST_TIME_FORMAT " twcc-time: %" GST_TIME_FORMAT " packets: %d", GST_TIME_ARGS (current_time), GST_TIME_ARGS (twcc->next_feedback_send_time), twcc->recv_packets->len); @@ -905,8 +1613,11 @@ rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint sender_ssrc, twcc->next_feedback_send_time += twcc->feedback_interval; /* Generate feedback, if there is some to send */ - if (twcc->recv_packets->len > 0) - rtp_twcc_manager_create_feedback (twcc); + if (twcc->recv_packets->len > 0) { + g_mutex_lock (&twcc->recv_lock); + rtp_twcc_manager_create_feedback_unlocked (twcc); + g_mutex_unlock (&twcc->recv_lock); + } } buf = g_queue_pop_head (twcc->rtcp_buffers); @@ -919,12 +1630,57 @@ rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint sender_ssrc, return buf; } +static guint8 +get_twcc_ext_id_for_pt (RTPTWCCManager * twcc, guint8 pt) +{ + guint8 twcc_ext_id; + GstCaps *caps; + gpointer value; + + value = g_hash_table_lookup (twcc->pt_to_twcc_ext_id, GUINT_TO_POINTER (pt)); + if (value) + return GPOINTER_TO_UINT (value); + + if (!twcc->caps_cb) + return 0; + + caps = twcc->caps_cb (pt, twcc->caps_ud); + if (!caps) + return 0; + + twcc_ext_id = + gst_rtp_get_extmap_id_for_attribute (gst_caps_get_structure (caps, 0), + TWCC_EXTMAP_STR); + gst_caps_unref (caps); + + g_hash_table_insert (twcc->pt_to_twcc_ext_id, GUINT_TO_POINTER (pt), + GUINT_TO_POINTER (twcc_ext_id)); + GST_LOG ("Added payload (%u) for twcc send ext-id: %u", pt, twcc_ext_id); + + return twcc_ext_id; +} + void rtp_twcc_manager_send_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) { + guint8 pinfo_twcc_ext_id; + + pinfo_twcc_ext_id = get_twcc_ext_id_for_pt (twcc, pinfo->pt); + + /* save the first valid twcc extid we get from pt */ + if (pinfo_twcc_ext_id > 0 && twcc->send_ext_id == 0) { + twcc->send_ext_id = pinfo_twcc_ext_id; + GST_INFO ("TWCC enabled for send using extension id: %u", + twcc->send_ext_id); + } + if (twcc->send_ext_id == 0) return; + /* the packet info twcc_ext_id should match the parsed one */ + if (pinfo_twcc_ext_id != twcc->send_ext_id) + return; + rtp_twcc_manager_set_send_twcc_seqnum (twcc, pinfo); } @@ -936,7 +1692,7 @@ rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, guint64 buffer_id, guint16 seqnum = (guint16) buffer_id; SentPacket *first = NULL; SentPacket *pkt = NULL; - guint idx; + gint idx; first = &g_array_index (twcc->sent_packets, SentPacket, 0); if (first == NULL) { @@ -944,57 +1700,59 @@ rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, guint64 buffer_id, return; } - idx = seqnum - first->seqnum; - - if (idx < twcc->sent_packets->len) { + idx = gst_rtp_buffer_compare_seqnum (first->seqnum, seqnum); + if (idx < 0) { + pkt = + &g_array_index (twcc->sent_packets, SentPacket, + twcc->sent_packets->len - 1); + } else if (idx < twcc->sent_packets->len) { pkt = &g_array_index (twcc->sent_packets, SentPacket, idx); - if (pkt && pkt->seqnum == seqnum) { - pkt->socket_ts = ts; - GST_LOG ("packet #%u, setting socket-ts %" GST_TIME_FORMAT, - seqnum, GST_TIME_ARGS (ts)); - } + } + + if (pkt && pkt->seqnum == seqnum) { + pkt->socket_ts = ts; + GST_LOG ("packet #%u, setting socket-ts %" GST_TIME_FORMAT, + seqnum, GST_TIME_ARGS (ts)); } else { GST_WARNING ("Unable to update send-time for twcc-seqnum #%u", seqnum); } } static void -_add_twcc_packet (GArray * twcc_packets, guint16 seqnum, guint status) +_add_parsed_packet (GArray * parsed_packets, guint16 seqnum, guint status) { - RTPTWCCPacket packet; - memset (&packet, 0, sizeof (RTPTWCCPacket)); - packet.local_ts = GST_CLOCK_TIME_NONE; - packet.remote_ts = GST_CLOCK_TIME_NONE; - packet.local_delta = GST_CLOCK_STIME_NONE; - packet.remote_delta = GST_CLOCK_STIME_NONE; - packet.delta_delta = GST_CLOCK_STIME_NONE; + ParsedPacket packet; packet.seqnum = seqnum; packet.status = status; - g_array_append_val (twcc_packets, packet); + packet.remote_ts = GST_CLOCK_TIME_NONE; + g_array_append_val (parsed_packets, packet); + GST_LOG ("Adding parsed packet #%u with status %u", seqnum, status); } static guint -_parse_run_length_chunk (GstBitReader * reader, GArray * twcc_packets, +_parse_run_length_chunk (GstBitReader * reader, GArray * parsed_packets, guint16 seqnum_offset, guint remaining_packets) { - guint16 run_length; - guint8 status_code; + guint16 run_length = 0; + guint8 status_code = 0; guint i; gst_bit_reader_get_bits_uint8 (reader, &status_code, 2); gst_bit_reader_get_bits_uint16 (reader, &run_length, 13); run_length = MIN (remaining_packets, run_length); + GST_LOG ("Found run-length: %u from seqnum: #%u with status code: %u", + run_length, seqnum_offset, status_code); for (i = 0; i < run_length; i++) { - _add_twcc_packet (twcc_packets, seqnum_offset + i, status_code); + _add_parsed_packet (parsed_packets, seqnum_offset + i, status_code); } return run_length; } static guint -_parse_status_vector_chunk (GstBitReader * reader, GArray * twcc_packets, +_parse_status_vector_chunk (GstBitReader * reader, GArray * parsed_packets, guint16 seqnum_offset, guint remaining_packets) { guint8 symbol_size; @@ -1005,38 +1763,30 @@ _parse_status_vector_chunk (GstBitReader * reader, GArray * twcc_packets, symbol_size += 1; num_bits = MIN (remaining_packets, 14 / symbol_size); + GST_LOG ("Found status vector for %u from seqnum: #%u with symbol size: %u", + num_bits, seqnum_offset, symbol_size); + for (i = 0; i < num_bits; i++) { guint8 status_code; if (gst_bit_reader_get_bits_uint8 (reader, &status_code, symbol_size)) - _add_twcc_packet (twcc_packets, seqnum_offset + i, status_code); + _add_parsed_packet (parsed_packets, seqnum_offset + i, status_code); } return num_bits; } -/* Remove all locally stored packets that has been reported - back to us */ +/* keep the last 1000 packets... FIXME: do something smarter */ static void -_prune_sent_packets (RTPTWCCManager * twcc, GArray * twcc_packets) +_prune_sent_packets (RTPTWCCManager * twcc) { - SentPacket *first; - RTPTWCCPacket *last; - guint16 last_idx; - - if (twcc_packets->len == 0 || twcc->sent_packets->len == 0) - return; - - first = &g_array_index (twcc->sent_packets, SentPacket, 0); - last = &g_array_index (twcc_packets, RTPTWCCPacket, twcc_packets->len - 1); - - last_idx = last->seqnum - first->seqnum; - - if (last_idx < twcc->sent_packets->len) - g_array_remove_range (twcc->sent_packets, 0, last_idx); + if (twcc->sent_packets->len > 1000) { + g_array_remove_range (twcc->sent_packets, 0, + twcc->sent_packets->len - 1000); + } } static void -_check_for_lost_packets (RTPTWCCManager * twcc, GArray * twcc_packets, +_check_for_lost_packets (RTPTWCCManager * twcc, GArray * parsed_packets, guint16 base_seqnum, guint16 packet_count, guint8 fb_pkt_count) { guint packets_lost; @@ -1071,12 +1821,12 @@ _check_for_lost_packets (RTPTWCCManager * twcc, GArray * twcc_packets, if (base_seqnum < twcc->expected_parsed_seqnum) { GST_DEBUG ("twcc seqnum is older than expected (%u < %u)", base_seqnum, twcc->expected_parsed_seqnum); - return; + goto done; } packets_lost = base_seqnum - twcc->expected_parsed_seqnum; for (i = 0; i < packets_lost; i++) { - _add_twcc_packet (twcc_packets, twcc->expected_parsed_seqnum + i, + _add_parsed_packet (parsed_packets, twcc->expected_parsed_seqnum + i, RTP_TWCC_PACKET_STATUS_NOT_RECV); } @@ -1086,11 +1836,24 @@ _check_for_lost_packets (RTPTWCCManager * twcc, GArray * twcc_packets, return; } -GArray * +static void +_add_parsed_packet_to_value_array (GValueArray * array, ParsedPacket * pkt) +{ + _append_structure_to_value_array (array, + gst_structure_new ("RTPTWCCPacket", + "seqnum", G_TYPE_UINT, pkt->seqnum, + "remote-ts", G_TYPE_UINT64, pkt->remote_ts, + "lost", G_TYPE_BOOLEAN, !GST_CLOCK_TIME_IS_VALID (pkt->remote_ts), + NULL)); +} + +GstStructure * rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, - guint8 * fci_data, guint fci_length) + guint8 * fci_data, guint fci_length, GstClockTime current_time) { - GArray *twcc_packets; + GstStructure *ret; + GValueArray *array; + guint16 base_seqnum; guint16 packet_count; GstClockTime base_time; @@ -1100,12 +1863,16 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, guint fci_parsed; guint i; SentPacket *first_sent_pkt = NULL; + GstClockTimeDiff rtt = GST_CLOCK_STIME_NONE; if (fci_length < 10) { GST_WARNING ("Malformed TWCC RTCP feedback packet"); return NULL; } + ret = gst_structure_new_empty ("RTPTWCCPackets"); + array = g_value_array_new (0); + base_seqnum = GST_READ_UINT16_BE (&fci_data[0]); packet_count = GST_READ_UINT16_BE (&fci_data[2]); base_time = GST_READ_UINT24_BE (&fci_data[4]) * REF_TIME_UNIT; @@ -1115,10 +1882,9 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, "base_time %" GST_TIME_FORMAT " fb_pkt_count: %u", base_seqnum, packet_count, GST_TIME_ARGS (base_time), fb_pkt_count); - twcc_packets = g_array_sized_new (FALSE, FALSE, - sizeof (RTPTWCCPacket), packet_count); + g_array_set_size (twcc->parsed_packets, 0); - _check_for_lost_packets (twcc, twcc_packets, + _check_for_lost_packets (twcc, twcc->parsed_packets, base_seqnum, packet_count, fb_pkt_count); fci_parsed = 8; @@ -1129,13 +1895,15 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, guint remaining_packets = packet_count - packets_parsed; gst_bit_reader_get_bits_uint8 (&reader, &chunk_type, 1); + GST_LOG ("Started parse for chunk-type: %u parsed packets: (%u/%u)", + chunk_type, packets_parsed, packet_count); if (chunk_type == RTP_TWCC_CHUNK_TYPE_RUN_LENGTH) { packets_parsed += _parse_run_length_chunk (&reader, - twcc_packets, seqnum_offset, remaining_packets); + twcc->parsed_packets, seqnum_offset, remaining_packets); } else { packets_parsed += _parse_status_vector_chunk (&reader, - twcc_packets, seqnum_offset, remaining_packets); + twcc->parsed_packets, seqnum_offset, remaining_packets); } fci_parsed += 2; } @@ -1144,8 +1912,8 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, first_sent_pkt = &g_array_index (twcc->sent_packets, SentPacket, 0); ts_rounded = base_time; - for (i = 0; i < twcc_packets->len; i++) { - RTPTWCCPacket *pkt = &g_array_index (twcc_packets, RTPTWCCPacket, i); + for (i = 0; i < twcc->parsed_packets->len; i++) { + ParsedPacket *pkt = &g_array_index (twcc->parsed_packets, ParsedPacket, i); gint16 delta = 0; GstClockTimeDiff delta_ts; @@ -1159,7 +1927,7 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, if (fci_parsed > fci_length) { GST_WARNING ("Malformed TWCC RTCP feedback packet"); - g_array_set_size (twcc_packets, 0); + g_array_set_size (twcc->parsed_packets, 0); break; } @@ -1173,29 +1941,95 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, " status: %u", pkt->seqnum, GST_TIME_ARGS (pkt->remote_ts), GST_STIME_ARGS (delta_ts), pkt->status); + _add_parsed_packet_to_value_array (array, pkt); } if (first_sent_pkt) { SentPacket *found = NULL; - guint16 sent_idx = pkt->seqnum - first_sent_pkt->seqnum; + guint16 sent_idx = + gst_rtp_buffer_compare_seqnum (first_sent_pkt->seqnum, pkt->seqnum); if (sent_idx < twcc->sent_packets->len) found = &g_array_index (twcc->sent_packets, SentPacket, sent_idx); if (found && found->seqnum == pkt->seqnum) { - if (GST_CLOCK_TIME_IS_VALID (found->socket_ts)) { - pkt->local_ts = found->socket_ts; - } else { - pkt->local_ts = found->ts; - } - pkt->size = found->size; - pkt->pt = found->pt; + found->remote_ts = pkt->remote_ts; GST_LOG ("matching pkt: #%u with local_ts: %" GST_TIME_FORMAT - " size: %u", pkt->seqnum, GST_TIME_ARGS (pkt->local_ts), pkt->size); + " size: %u, remote-ts: %" GST_TIME_FORMAT, pkt->seqnum, + GST_TIME_ARGS (found->local_ts), + found->size * 8, GST_TIME_ARGS (pkt->remote_ts)); + + _add_packet_to_stats (twcc, pkt, found); + + /* calculate the round-trip time */ + rtt = GST_CLOCK_DIFF (found->local_ts, current_time); + } } } - _prune_sent_packets (twcc, twcc_packets); + if (GST_CLOCK_STIME_IS_VALID (rtt)) + twcc->avg_rtt = WEIGHT (rtt, twcc->avg_rtt, 0.1); + twcc->last_report_time = current_time; + + _prune_sent_packets (twcc); - return twcc_packets; + _structure_take_value_array (ret, "packets", array); + + return ret; } + +GstStructure * +rtp_twcc_manager_get_windowed_stats (RTPTWCCManager * twcc, + GstClockTime stats_window_size, GstClockTime stats_window_delay) +{ + GstStructure *ret; + GValueArray *array; + GHashTableIter iter; + gpointer key; + gpointer value; + GstClockTimeDiff start_time; + GstClockTimeDiff end_time; + + GstClockTime last_ts = twcc_stats_ctx_get_last_local_ts (twcc->stats_ctx); + if (!GST_CLOCK_TIME_IS_VALID (last_ts)) + return twcc_stats_ctx_get_structure (twcc->stats_ctx);; + + array = g_value_array_new (0); + end_time = GST_CLOCK_DIFF (stats_window_delay, last_ts); + start_time = end_time - stats_window_size; + + GST_DEBUG_OBJECT (twcc, + "Calculating windowed stats for the window %" GST_STIME_FORMAT + " starting from %" GST_STIME_FORMAT " to: %" GST_STIME_FORMAT, + GST_STIME_ARGS (stats_window_size), GST_STIME_ARGS (start_time), + GST_STIME_ARGS (end_time)); + + twcc_stats_ctx_calculate_windowed_stats (twcc->stats_ctx, start_time, + end_time); + ret = twcc_stats_ctx_get_structure (twcc->stats_ctx); + GST_LOG ("Full stats: %" GST_PTR_FORMAT, ret); + + g_hash_table_iter_init (&iter, twcc->stats_ctx_by_pt); + while (g_hash_table_iter_next (&iter, &key, &value)) { + GstStructure *s; + guint pt = GPOINTER_TO_UINT (key); + TWCCStatsCtx *ctx = value; + twcc_stats_ctx_calculate_windowed_stats (ctx, start_time, end_time); + s = twcc_stats_ctx_get_structure (ctx); + gst_structure_set (s, "pt", G_TYPE_UINT, pt, NULL); + _append_structure_to_value_array (array, s); + GST_LOG ("Stats for pt %u: %" GST_PTR_FORMAT, pt, s); + } + + _structure_take_value_array (ret, "payload-stats", array); + + return ret; +} + +void +rtp_twcc_manager_set_callback (RTPTWCCManager * twcc, RTPTWCCManagerCaps cb, + gpointer user_data) +{ + twcc->caps_cb = cb; + twcc->caps_ud = user_data; +} \ No newline at end of file diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h index bc8af8c4a19..d15e6976265 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h @@ -25,32 +25,20 @@ #include #include "rtpstats.h" -typedef struct _RTPTWCCPacket RTPTWCCPacket; -typedef enum _RTPTWCCPacketStatus RTPTWCCPacketStatus; - G_DECLARE_FINAL_TYPE (RTPTWCCManager, rtp_twcc_manager, RTP, TWCC_MANAGER, GObject) #define RTP_TYPE_TWCC_MANAGER (rtp_twcc_manager_get_type()) #define RTP_TWCC_MANAGER_CAST(obj) ((RTPTWCCManager *)(obj)) -enum _RTPTWCCPacketStatus -{ - RTP_TWCC_PACKET_STATUS_NOT_RECV = 0, - RTP_TWCC_PACKET_STATUS_SMALL_DELTA = 1, - RTP_TWCC_PACKET_STATUS_LARGE_NEGATIVE_DELTA = 2, -}; - -struct _RTPTWCCPacket -{ - GstClockTime local_ts; - GstClockTime remote_ts; - GstClockTimeDiff local_delta; - GstClockTimeDiff remote_delta; - GstClockTimeDiff delta_delta; - RTPTWCCPacketStatus status; - guint16 seqnum; - guint size; - guint8 pt; -}; +/** + * RTPTWCCManagerCaps: + * @payload: the payload + * @user_data: user data specified when registering + * + * This callback will be called when @twcc needs the caps of @payload. + * + * Returns: the caps of @payload. + */ +typedef GstCaps * (*RTPTWCCManagerCaps) (guint8 payload, gpointer user_data); RTPTWCCManager * rtp_twcc_manager_new (guint mtu); @@ -74,7 +62,13 @@ void rtp_twcc_manager_send_packet (RTPTWCCManager * twcc, GstBuffer * rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint32 sender_ssrc, GstClockTime current_time); -GArray * rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, - guint8 * fci_data, guint fci_length); +GstStructure * rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, + guint8 * fci_data, guint fci_length, GstClockTime current_time); + +GstStructure * rtp_twcc_manager_get_windowed_stats (RTPTWCCManager * twcc, + GstClockTime stats_window_size, GstClockTime stats_window_delay); + +void rtp_twcc_manager_set_callback (RTPTWCCManager * twcc, + RTPTWCCManagerCaps cb, gpointer user_data); #endif /* __RTP_TWCC_H__ */ diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 23de2d8a5be..556f34b7583 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -29,9 +29,20 @@ #include #include +#include #include #include +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_VALGRIND +# include +#else +# define RUNNING_ON_VALGRIND 0 +#endif + #define TEST_BUF_CLOCK_RATE 8000 #define TEST_BUF_PT 0 #define TEST_BUF_SSRC 0x01BADBAD @@ -44,12 +55,15 @@ #define TEST_TWCC_EXT_ID 5 #define TWCC_EXTMAP_STR "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" +#define TEST_RTX_BUF_PT 100 +#define TEST_RTX_BUF_SSRC 0xd3add3ad + static GstCaps * -generate_caps (void) +generate_caps (guint8 payload_type) { return gst_caps_new_simple ("application/x-rtp", "clock-rate", G_TYPE_INT, TEST_BUF_CLOCK_RATE, - "payload", G_TYPE_INT, TEST_BUF_PT, NULL); + "payload", G_TYPE_INT, payload_type, NULL); } static GstBuffer * @@ -112,7 +126,7 @@ generate_twcc_send_buffer_full (guint seqnum, gboolean marker_bit, { return generate_test_buffer_full (seqnum * TEST_BUF_DURATION, seqnum, seqnum * TEST_RTP_TS_DURATION, ssrc, marker_bit, - payload_type, TEST_TWCC_EXT_ID, seqnum); + payload_type, 0, 0); } static GstBuffer * @@ -122,6 +136,85 @@ generate_twcc_send_buffer (guint seqnum, gboolean marker_bit) TEST_BUF_PT); } +static GstBuffer * +generate_rtx_buffer (guint rtx_seqnum, GstBuffer * buffer) +{ + GstMemory *mem = NULL; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstRTPBuffer new_rtp = GST_RTP_BUFFER_INIT; + GstBuffer *new_buffer = gst_buffer_new (); + GstMapInfo map; + + gst_rtp_buffer_map (buffer, GST_MAP_READ, &rtp); + gst_rtp_buffer_get_payload (&rtp); + + /* copy fixed header */ + mem = gst_memory_copy (rtp.map[0].memory, 0, rtp.size[0]); + gst_buffer_append_memory (new_buffer, mem); + + /* copy extension if any */ + if (rtp.size[1]) { + mem = gst_allocator_alloc (NULL, rtp.size[1], NULL); + gst_memory_map (mem, &map, GST_MAP_WRITE); + memcpy (map.data, rtp.data[1], rtp.size[1]); + gst_memory_unmap (mem, &map); + gst_buffer_append_memory (new_buffer, mem); + } + + /* copy payload and add OSN just before */ + mem = gst_allocator_alloc (NULL, 2 + rtp.size[2], NULL); + + gst_memory_map (mem, &map, GST_MAP_WRITE); + GST_WRITE_UINT16_BE (map.data, gst_rtp_buffer_get_seq (&rtp)); + if (rtp.size[2]) + memcpy (map.data + 2, rtp.data[2], rtp.size[2]); + gst_memory_unmap (mem, &map); + gst_buffer_append_memory (new_buffer, mem); + + /* everything needed is copied */ + gst_rtp_buffer_unmap (&rtp); + + gst_rtp_buffer_map (new_buffer, GST_MAP_WRITE, &new_rtp); + gst_rtp_buffer_set_payload_type (&new_rtp, TEST_RTX_BUF_PT); + gst_rtp_buffer_set_ssrc (&new_rtp, TEST_RTX_BUF_SSRC); + gst_rtp_buffer_set_seq (&new_rtp, rtx_seqnum); + gst_rtp_buffer_unmap (&new_rtp); + + /* Copy over timestamps */ + gst_buffer_copy_into (new_buffer, buffer, GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + + /* mark this is a RETRANSMISSION buffer */ + GST_BUFFER_FLAG_SET (new_buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION); + + return new_buffer; +} + +static gint32 +read_twcc_seqnum (GstBuffer * buf, guint8 twcc_ext_id) +{ + guint16 twcc_seqnum; + gpointer ext_data; + guint ext_size; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + if (!gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp)) + return -1; + + if (!gst_rtp_buffer_get_extension_onebyte_header (&rtp, twcc_ext_id, + 0, &ext_data, &ext_size)) { + gst_rtp_buffer_unmap (&rtp); + return -1; + } + + fail_unless (ext_data != NULL); + fail_unless (ext_size == 2); + + twcc_seqnum = GST_READ_UINT16_BE (ext_data); + gst_rtp_buffer_unmap (&rtp); + + return twcc_seqnum; +} + typedef struct { GstHarness *send_rtp_h; @@ -133,15 +226,23 @@ typedef struct GstTestClock *testclock; GstCaps *caps; + GHashTable *pt_to_caps_map; + gboolean running; GMutex lock; GstStructure *last_twcc_stats; } SessionHarness; static GstCaps * -_pt_map_requested (GstElement * element, guint pt, gpointer data) +_pt_map_requested (G_GNUC_UNUSED GstElement * element, guint pt, gpointer data) { SessionHarness *h = data; + GstCaps *caps = g_hash_table_lookup (h->pt_to_caps_map, + GUINT_TO_POINTER (pt)); + + if (caps) + return gst_caps_copy (caps); + return gst_caps_copy (h->caps); } @@ -161,23 +262,32 @@ _notify_twcc_stats (GParamSpec * spec G_GNUC_UNUSED, } static GstStructure * -session_harness_get_last_twcc_stats (SessionHarness * h) +session_harness_get_twcc_stats_full (SessionHarness * h, + GstClockTime stats_window_size, GstClockTime stats_window_delay) { - GstStructure *ret = NULL; - g_mutex_lock (&h->lock); - if (h->last_twcc_stats) - ret = gst_structure_copy (h->last_twcc_stats); - g_mutex_unlock (&h->lock); + GstStructure *ret; + g_signal_emit_by_name (h->internal_session, "get-twcc-windowed-stats", + stats_window_size, stats_window_delay, &ret); return ret; } +static GstStructure * +session_harness_get_twcc_stats (SessionHarness * h) +{ + return session_harness_get_twcc_stats_full (h, + 300 * GST_MSECOND, 200 * GST_MSECOND); +} + static SessionHarness * session_harness_new (void) { SessionHarness *h = g_new0 (SessionHarness, 1); - h->caps = generate_caps (); + h->caps = generate_caps (TEST_BUF_PT); g_mutex_init (&h->lock); + h->pt_to_caps_map = + g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify) gst_caps_unref); + h->testclock = GST_TEST_CLOCK_CAST (gst_test_clock_new ()); gst_system_clock_set_default (GST_CLOCK_CAST (h->testclock)); @@ -215,6 +325,8 @@ session_harness_free (SessionHarness * h) gst_caps_unref (h->caps); gst_object_unref (h->testclock); + g_hash_table_destroy (h->pt_to_caps_map); + gst_harness_teardown (h->rtcp_h); gst_harness_teardown (h->recv_rtp_h); gst_harness_teardown (h->send_rtp_h); @@ -352,12 +464,38 @@ session_harness_set_twcc_recv_ext_id (SessionHarness * h, guint8 ext_id) g_signal_emit_by_name (h->session, "clear-pt-map"); } +static GstStructure * +create_rtx_map (const gchar * name, guint key, guint value) +{ + gchar *key_str = g_strdup_printf ("%u", key); + GstStructure *s = gst_structure_new (name, + key_str, G_TYPE_UINT, (guint) value, NULL); + g_free (key_str); + return s; +} + static void -session_harness_set_twcc_send_ext_id (SessionHarness * h, guint8 ext_id) +session_harness_enable_rtx (SessionHarness * h) { - GstCaps *caps = gst_caps_copy (h->caps); - _add_twcc_field_to_caps (caps, ext_id); - gst_harness_set_src_caps (h->send_rtp_h, caps); + GstStructure *rtx_map = + create_rtx_map ("rtx-map", TEST_BUF_SSRC, TEST_RTX_BUF_SSRC); + g_object_set (h->internal_session, "rtx-ssrc-map", rtx_map, NULL); + gst_structure_free (rtx_map); +} + +static void +session_harness_add_caps_for_pt (SessionHarness * h, GstCaps * caps, guint8 pt) +{ + g_hash_table_insert (h->pt_to_caps_map, GUINT_TO_POINTER (pt), caps); + g_signal_emit_by_name (h->session, "clear-pt-map"); +} + +static void +session_harness_add_twcc_caps_for_pt (SessionHarness * h, guint8 pt) +{ + GstCaps *caps = generate_caps (pt); + _add_twcc_field_to_caps (caps, TEST_TWCC_EXT_ID); + session_harness_add_caps_for_pt (h, caps, pt); } GST_START_TEST (test_multiple_ssrc_rr) @@ -672,7 +810,7 @@ GST_START_TEST (test_internal_sources_timeout) gst_buffer_unref (buf); /* ok, now let's push some RTP packets */ - caps = generate_caps (); + caps = generate_caps (TEST_BUF_PT); gst_caps_set_simple (caps, "ssrc", G_TYPE_UINT, 0x01BADBAD, "rtx-ssrc", G_TYPE_UINT, 0x01020304, NULL); @@ -1640,7 +1778,7 @@ GST_START_TEST (test_request_fir_after_pli_in_caps) /* Rebuild the caps */ gst_caps_unref (h->caps); - h->caps = generate_caps (); + h->caps = generate_caps (TEST_BUF_PT); /* add FIR-capabilites to our caps */ gst_caps_set_simple (h->caps, "rtcp-fb-ccm-fir", G_TYPE_BOOLEAN, TRUE, NULL); @@ -2877,48 +3015,53 @@ G_STMT_START { \ guint j = 0; \ GValueArray *packets_array = g_value_get_boxed ( \ gst_structure_get_value (gst_event_get_structure ((event)), "packets")); \ + g_assert (packets_array); \ for (i = 0; i < packets_array->n_values; i++) { \ - TWCCPacket *twcc_pkt; \ - GstClockTime ts; \ - guint seqnum; \ - gboolean lost; \ + TWCCPacket *twcc_pkt = NULL; \ + GstClockTime ts = 0; \ + guint seqnum = 0; \ + gboolean lost = FALSE; \ const GstStructure *pkt_s = \ gst_value_get_structure (g_value_array_get_nth (packets_array, i)); \ + g_assert (pkt_s); \ fail_unless (gst_structure_get_boolean (pkt_s, "lost", &lost)); \ - if (lost) \ - continue; \ - fail_unless (gst_structure_get_clock_time (pkt_s, "remote-ts", &ts)); \ - fail_unless (gst_structure_get_uint (pkt_s, "seqnum", &seqnum)); \ - twcc_pkt = &(packets)[j++]; \ - fail_unless_equals_int (twcc_pkt->seqnum, seqnum); \ - fail_unless_equals_twcc_clocktime (twcc_pkt->timestamp, ts); \ + if (!lost) { \ + fail_unless (gst_structure_get_clock_time (pkt_s, "remote-ts", &ts)); \ + fail_unless (gst_structure_get_uint (pkt_s, "seqnum", &seqnum)); \ + if (j < G_N_ELEMENTS (packets)) { \ + twcc_pkt = &(packets[j]); \ + j++; \ + } \ + if (twcc_pkt) { \ + fail_unless_equals_int (twcc_pkt->seqnum, seqnum); \ + fail_unless_equals_twcc_clocktime (twcc_pkt->timestamp, ts); \ + } \ + } \ } \ gst_event_unref (event); \ } G_STMT_END -#define twcc_verify_packets_to_packets(send_h, recv_h, packets) \ +#define twcc_verify_packets_to_packets(send_h, recv_h, recv_pkt, parsed_pkt) \ G_STMT_START { \ guint i; \ GstEvent *event; \ - twcc_push_packets ((recv_h), packets); \ + twcc_push_packets ((recv_h), recv_pkt); \ session_harness_recv_rtcp ((send_h), \ session_harness_produce_twcc ((recv_h))); \ for (i = 0; i < 2; i++) \ gst_event_unref (gst_harness_pull_upstream_event ((send_h)->send_rtp_h)); \ event = gst_harness_pull_upstream_event ((send_h)->send_rtp_h); \ - twcc_verify_packets_to_event (packets, event); \ + twcc_verify_packets_to_event (parsed_pkt, event); \ } G_STMT_END -#define twcc_verify_stats(h, bitrate_sent, bitrate_recv, pkts_sent, pkts_recv, loss_pct, avg_dod) \ +#define twcc_verify_stats(twcc_stats, bitrate_sent, bitrate_recv, pkts_sent, pkts_recv, loss_pct, avg_dod) \ G_STMT_START { \ - GstStructure *twcc_stats; \ guint stats_bitrate_sent; \ guint stats_bitrate_recv; \ guint stats_packets_sent; \ guint stats_packets_recv; \ gdouble stats_loss_pct; \ GstClockTimeDiff stats_avg_dod; \ - twcc_stats = session_harness_get_last_twcc_stats (h); \ fail_unless (gst_structure_get (twcc_stats, \ "bitrate-sent", G_TYPE_UINT, &stats_bitrate_sent, \ "bitrate-recv", G_TYPE_UINT, &stats_bitrate_recv, \ @@ -2932,7 +3075,22 @@ G_STMT_START { fail_unless_equals_int (pkts_recv, stats_packets_recv); \ fail_unless_equals_float (loss_pct, stats_loss_pct); \ fail_unless_equals_int64 (avg_dod, stats_avg_dod); \ - gst_structure_free (twcc_stats); \ +} G_STMT_END + +#define twcc_send_recv_buffers(h_send, h_recv, buffers) \ +G_STMT_START { \ + guint i; \ + session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); \ + for (i = 0; i < G_N_ELEMENTS(buffers); i++) { \ + GstBuffer *buf = buffers[i]; \ + GstFlowReturn res = session_harness_send_rtp (h_send, buf); \ + fail_unless_equals_int (GST_FLOW_OK, res); \ + buf = session_harness_pull_send_rtp (h_send); \ + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); \ + res = session_harness_recv_rtp (h_recv, buf); \ + fail_unless_equals_int (GST_FLOW_OK, res); \ + } \ + session_harness_recv_rtcp (h_send, session_harness_produce_twcc (h_recv)); \ } G_STMT_END GST_START_TEST (test_twcc_1_bit_status_vector) @@ -2972,7 +3130,7 @@ GST_START_TEST (test_twcc_1_bit_status_vector) twcc_verify_packets_to_fci (h0, packets, exp_fci); /* and check we can parse this back to the original packets */ - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3024,7 +3182,7 @@ GST_START_TEST (test_twcc_status_vector_split_large_delta) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3060,7 +3218,7 @@ GST_START_TEST (test_twcc_2_bit_status_vector) twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3118,7 +3276,7 @@ GST_START_TEST (test_twcc_status_vector_split_with_gap) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3199,7 +3357,7 @@ GST_START_TEST (test_twcc_status_vector_split_into_three) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3232,7 +3390,7 @@ GST_START_TEST (test_twcc_2_bit_full_status_vector) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3250,7 +3408,7 @@ GST_START_TEST (test_twcc_various_gaps) {seq, seq * 250 * GST_USECOND, TRUE}, }; - twcc_verify_packets_to_packets (h, h, packets); + twcc_verify_packets_to_packets (h, h, packets, packets); session_harness_free (h); } @@ -3284,7 +3442,7 @@ GST_START_TEST (test_twcc_negative_delta) twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3318,7 +3476,7 @@ GST_START_TEST (test_twcc_seqnum_wrap) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3447,7 +3605,7 @@ GST_START_TEST (test_twcc_huge_seqnum_gap) twcc_push_packets (h0, packets); twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3493,6 +3651,99 @@ GST_START_TEST (test_twcc_duplicate_seqnums) GST_END_TEST; +GST_START_TEST (test_twcc_duplicate_previous_seqnums) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + + /* We receive pkt #1 again and this last one should be ignored */ + TWCCPacket packets[] = { + {1, 4 * 32 * GST_MSECOND, FALSE}, + {2, 5 * 32 * GST_MSECOND, FALSE}, + {1, 6 * 32 * GST_MSECOND, FALSE}, + {3, 7 * 32 * GST_MSECOND, TRUE}, + }; + + guint8 exp_fci[] = { + 0x00, 0x01, /* base sequence number: 1 */ + 0x00, 0x03, /* packet status count: 2 */ + 0x00, 0x00, 0x02, /* reference time: 2 * 64ms */ + 0x00, /* feedback packet count: 0 */ + /* packet chunks: */ + 0xd6, 0x00, /* 1 1 0 1 0 1 1 0 | 0 0 0 0 0 0 0 0 */ + 0x00, 0x80, /* recv deltas: +0, +32ms, + 64ms */ + 0x01, 0x00, + 0x00, 0x00, /* padding */ + }; + + twcc_push_packets (h, packets); + + buf = session_harness_produce_twcc (h); + twcc_verify_fci (buf, exp_fci); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_missing_packet_duplicates_last) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + + /* Verify the behavior that if we have had a packetloss in the + previous report, we start the next report with the last packet + of the previous report */ + + /* first we have a gap of 12 packets */ + TWCCPacket packets0[] = { + {10, 0 * 250 * GST_USECOND, FALSE}, + {22, 1 * 250 * GST_USECOND, TRUE}, + }; + + /* and then two "normal" ones */ + TWCCPacket packets1[] = { + {23, 2 * 250 * GST_USECOND, FALSE}, + {24, 3 * 250 * GST_USECOND, TRUE}, + }; + + guint8 exp_fci0[] = { + 0x00, 0x0a, /* base sequence number: 10 */ + 0x00, 0x0d, /* packet status count: 13 */ + 0x00, 0x00, 0x00, /* reference time: 0 ms */ + 0x00, /* feedback packet count: 0 */ + /* packet chunks: */ + 0xa0, 0x02, /* */ + 0x00, 0x01, /* recv deltas: +0, +1 */ + }; + + guint8 exp_fci1[] = { + 0x00, 0x16, /* base sequence number: 22 */ + 0x00, 0x03, /* packet status count: 3 */ + 0x00, 0x00, 0x00, /* reference time: 0 ms */ + 0x01, /* feedback packet count: 1 */ + /* packet chunks: */ + 0x20, 0x03, /* */ + 0x01, 0x01, 0x01, /* recv deltas: +1, +1, +1 */ + 0x00, 0x00, 0x00, /* padding */ + }; + + twcc_push_packets (h, packets0); + buf = session_harness_produce_twcc (h); + twcc_verify_fci (buf, exp_fci0); + gst_buffer_unref (buf); + + twcc_push_packets (h, packets1); + buf = session_harness_produce_twcc (h); + twcc_verify_fci (buf, exp_fci1); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + GST_START_TEST (test_twcc_multiple_markers) { @@ -3768,8 +4019,7 @@ GST_START_TEST (test_twcc_double_gap) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3780,59 +4030,255 @@ GST_END_TEST; GST_START_TEST (test_twcc_recv_packets_reordered) { SessionHarness *h = session_harness_new (); - GstBuffer *buf; - /* a reordered seqence, with marker-bits for #3 and #4 */ - TWCCPacket packets[] = { - {1, 1 * 250 * GST_USECOND, FALSE} - , - {3, 2 * 250 * GST_USECOND, TRUE} - , - {2, 3 * 250 * GST_USECOND, FALSE} - , - {4, 4 * 250 * GST_USECOND, TRUE} - , + /* *INDENT-OFF* */ + TWCCPacket packets0[] = { + { 1, 1 * 250 * GST_USECOND, FALSE}, + { 2, 2 * 250 * GST_USECOND, FALSE}, + { 3, 3 * 250 * GST_USECOND, FALSE}, + { 4, 4 * 250 * GST_USECOND, FALSE}, + {10, 5 * 250 * GST_USECOND, TRUE}, }; - /* first we expect #2 to be reported lost */ + TWCCPacket packets1[] = { + { 5, 6 * 250 * GST_USECOND, FALSE}, + { 6, 7 * 250 * GST_USECOND, FALSE}, + { 7, 8 * 250 * GST_USECOND, FALSE}, + { 8, 9 * 250 * GST_USECOND, TRUE}, + }; + + TWCCPacket packets2[] = { + { 9, 10 * 250 * GST_USECOND, FALSE}, + { 11, 11 * 250 * GST_USECOND, TRUE}, + }; + /* *INDENT-ON* */ + + /* this reports 1 to 4, then 5 to 9 missing, and then 10 */ guint8 exp_fci0[] = { 0x00, 0x01, /* base sequence number: 1 */ - 0x00, 0x03, /* packet status count: 3 */ + 0x00, 0x0a, /* packet status count: 10 */ 0x00, 0x00, 0x00, /* reference time: 0 */ 0x00, /* feedback packet count: 0 */ /* packet chunks: */ - 0xa8, 0x00, /* 1 0 1 0 1 0 0 0 | 0 0 0 0 0 0 0 0 */ - 0x01, 0x01, /* recv deltas, +1, +1 */ + 0xbc, 0x10, + 0x01, 0x01, 0x01, 0x01, 0x01, + 0x00, /* padding */ }; - /* and then when 2 actually arrives, it is already reported lost, - so we will not re-report it, but drop it */ + /* this reports 5 to 8, 9 missing and then 10 (again) */ guint8 exp_fci1[] = { - 0x00, 0x04, /* base sequence number: 4 */ - 0x00, 0x01, /* packet status count: 1 */ + 0x00, 0x05, /* base sequence number: 5 */ + 0x00, 0x06, /* packet status count: 6 */ 0x00, 0x00, 0x00, /* reference time: 0 */ 0x01, /* feedback packet count: 1 */ /* packet chunks: */ - 0x20, 0x01, /* 0 0 1 0 0 0 0 0 | 0 0 0 0 0 0 0 1 */ - 0x04, /* recv deltas, +4 */ + 0xd5, 0x48, + 0x06, + 0x01, 0x01, 0x01, + 0xff, 0xfc, + }; + + /* this reports 9, 10 (again!) and 11 */ + guint8 exp_fci2[] = { + 0x00, 0x09, /* base sequence number: 4 */ + 0x00, 0x03, /* packet status count: 3 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ + 0x02, /* feedback packet count: 2 */ + /* packet chunks: */ + 0xd9, 0x00, + 0x0a, + 0xff, 0xfb, + 0x06, + 0x00, 0x00, /* padding */ + }; + + twcc_verify_packets_to_fci (h, packets0, exp_fci0); + twcc_verify_packets_to_fci (h, packets1, exp_fci1); + twcc_verify_packets_to_fci (h, packets2, exp_fci2); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_recv_packets_reordered_and_lost) +{ + SessionHarness *h0 = session_harness_new (); + SessionHarness *h1 = session_harness_new (); + + /* *INDENT-OFF* */ + TWCCPacket packets[] = { + { 1488, 557500000, FALSE}, + { 1494, 558500000, FALSE}, + { 1503, 555250000, FALSE}, + { 1504, 559500000, FALSE}, + { 1509, 595500000, TRUE}, + }; + /* *INDENT-ON* */ + + /* this reports 1 to 4, then 5 to 9 missing, and then 10 */ + guint8 exp_fci[] = { + 0x05, 0xd0, /* base sequence number: 1488 */ + 0x00, 0x16, /* packet status count: 22 */ + 0x00, 0x00, 0x08, /* reference time: 0 */ + 0x00, /* feedback packet count: 0 */ + /* packet chunks: */ + 0xd0, 0x01, /* 11 01 00 00 | 00 00 00 01 - packet 1488, then 5 missing, then 1494 */ + 0x00, 0x08, /* 00 00 00 00 | 00 00 10 00 - run length of 8 missing packets, 1495 to 1502 */ + 0xe4, 0x01, /* 11 10 01 00 | 00 00 00 01 - packets 1503, 1504 and 1509 */ + + 0xb6, 0x04, 0xff, 0xf3, 0x11, 0x90, + }; + + twcc_verify_packets_to_fci (h0, packets, exp_fci); + twcc_verify_packets_to_packets (h1, h1, packets, packets); + + session_harness_free (h0); + session_harness_free (h1); +} + +GST_END_TEST; + + +GST_START_TEST (test_twcc_recv_packets_reordered_within_report_interval) +{ + SessionHarness *h0 = session_harness_new (); + SessionHarness *h1 = session_harness_new (); + + /* a reordered seqence, all within the same report */ + TWCCPacket packets[] = { + {3, 250 * GST_USECOND, FALSE}, + {1, 500 * GST_USECOND, FALSE}, + {2, 750 * GST_USECOND, FALSE}, + {4, 1000 * GST_USECOND, TRUE}, + }; + + TWCCPacket reorderd_packets[] = { + {1, 500 * GST_USECOND, FALSE}, + {2, 750 * GST_USECOND, FALSE}, + {3, 250 * GST_USECOND, FALSE}, + {4, 1000 * GST_USECOND, TRUE}, + }; + + /* we expect this to be handled gracefully */ + guint8 exp_fci[] = { + 0x00, 0x01, /* base sequence number: 1 */ + 0x00, 0x04, /* packet status count: 3 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ + 0x00, /* feedback packet count: 0 */ + /* packet chunks: */ + 0xd6, 0x40, /* 11 - Status Vector Chunk, 2 bit + 01 - Packet received, small delta + 01 - Packet received, small delta + 10 - Packet received, large or negative delta + 01 - Packet received, small delta + */ + 0x02, /* + 500us - abs: 500us */ + 0x01, /* + 250us - abs: 750us */ + 0xff, 0xfe, /* - 500us - abs: 250us */ + 0x03, /* + 750us - abs: 1000us */ 0x00, /* padding */ }; - twcc_push_packets (h, packets); + twcc_verify_packets_to_fci (h0, packets, exp_fci); + twcc_verify_packets_to_packets (h1, h1, packets, reorderd_packets); - buf = session_harness_produce_twcc (h); - twcc_verify_fci (buf, exp_fci0); - gst_buffer_unref (buf); + session_harness_free (h0); + session_harness_free (h1); +} - buf = session_harness_produce_twcc (h); - twcc_verify_fci (buf, exp_fci1); - gst_buffer_unref (buf); +GST_END_TEST; - session_harness_free (h); +GST_START_TEST (test_twcc_reordering_send_recv) +{ + SessionHarness *h_send = session_harness_new (); + SessionHarness *h_recv = session_harness_new (); + GList *bufs = NULL; + GstBuffer *buf; + GstStructure *twcc_stats; + + g_object_set (h_recv->internal_session, "twcc-feedback-interval", + 50 * GST_MSECOND, NULL); + + /* enable twcc */ + session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); + session_harness_add_twcc_caps_for_pt (h_send, TEST_BUF_PT); + + /* sender sends the buffers */ + for (guint i = 0; i < 12; i++) { + fail_unless_equals_int (GST_FLOW_OK, session_harness_send_rtp (h_send, + generate_twcc_send_buffer (i, FALSE))); + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + /* get the buffer ready for the network */ + bufs = g_list_append (bufs, session_harness_pull_send_rtp (h_send)); + } + + /* reorder packet 10 to the 5th position */ + buf = g_list_nth_data (bufs, 10); + bufs = g_list_remove (bufs, buf); + bufs = g_list_insert (bufs, buf, 5); + + /* 1 - send packets 0,1,2,3,4,10(!) */ + for (guint i = 0; i < 6; i++) { + buf = g_list_nth_data (bufs, i); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h_recv, + buf)); + } + buf = session_harness_produce_twcc (h_recv); + session_harness_recv_rtcp (h_send, buf); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 0, 0, 0, 0, 0.0f, 0); + gst_structure_free (twcc_stats); + + /* 2 - send packets 5,6,7,8 */ + for (guint i = 6; i < 10; i++) { + buf = g_list_nth_data (bufs, i); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h_recv, + buf)); + } + buf = session_harness_produce_twcc (h_recv); + session_harness_recv_rtcp (h_send, buf); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 0, 0, 0, 0, 0.0f, 0); + gst_structure_free (twcc_stats); + + /* 3 - send packets 9,11 */ + for (guint i = 10; i < 12; i++) { + buf = g_list_nth_data (bufs, i); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h_recv, + buf)); + } + buf = session_harness_produce_twcc (h_recv); + session_harness_recv_rtcp (h_send, buf); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 532800, 532800, 2, 2, 0.0f, 0); + gst_structure_free (twcc_stats); + + /* now send enough buffers to get within our stats window */ + for (guint i = 12; i < 26; i++) { + fail_unless_equals_int (GST_FLOW_OK, session_harness_send_rtp (h_send, + generate_twcc_send_buffer (i, FALSE))); + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + buf = session_harness_pull_send_rtp (h_send); + session_harness_recv_rtp (h_recv, buf); + } + buf = session_harness_produce_twcc (h_recv); + session_harness_recv_rtcp (h_send, buf); + + /* we get stats for the first 16 packets, noticing no loss + even though packets were heavily reordered */ + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 532800, 532800, 16, 16, 0.0f, 0); + gst_structure_free (twcc_stats); + + g_list_free (bufs); + session_harness_free (h_send); + session_harness_free (h_recv); } GST_END_TEST; + GST_START_TEST (test_twcc_recv_late_packet_fb_pkt_count_wrap) { SessionHarness *h = session_harness_new (); @@ -3851,13 +4297,13 @@ GST_START_TEST (test_twcc_recv_late_packet_fb_pkt_count_wrap) }; guint8 exp_fci1[] = { - 0x01, 0x01, /* base sequence number: 257 */ + 0x00, 0xff, /* base sequence number: 255 */ 0x00, 0x01, /* packet status count: 1 */ - 0x00, 0x00, 0x01, /* reference time: 1 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ 0x01, /* feedback packet count: 1 */ /* packet chunks: */ 0x20, 0x01, /* 0 0 1 0 0 0 0 0 | 0 0 0 0 0 0 0 1 */ - 0x01, /* 1 recv-delta */ + 0xff, /* 1 recv-delta */ 0x00, /* padding */ }; @@ -3918,32 +4364,27 @@ GST_START_TEST (test_twcc_recv_rtcp_reordered) guint i; /* three frames, two packets each */ - TWCCPacket packets[] = { - {1, 1 * GST_SECOND, FALSE} - , - {2, 2 * GST_SECOND, TRUE} - , - {3, 3 * GST_SECOND, FALSE} - , - {4, 4 * GST_SECOND, TRUE} - , - {5, 5 * GST_SECOND, FALSE} - , - {6, 6 * GST_SECOND, TRUE} - , - {7, 7 * GST_SECOND, FALSE} - , - {8, 8 * GST_SECOND, TRUE} - , + TWCCPacket packets0[] = { + {1, 1 * GST_SECOND, FALSE}, + {2, 2 * GST_SECOND, TRUE}, }; - -/* - TWCCPacket expected_packets0[] = { - {1, 1 * 250 * GST_USECOND, FALSE}, - {2, 2 * 250 * GST_USECOND, TRUE}, + TWCCPacket packets1[] = { + {3, 3 * GST_SECOND, FALSE}, + {4, 4 * GST_SECOND, TRUE}, + }; + TWCCPacket packets2[] = { + {5, 5 * GST_SECOND, FALSE}, + {6, 6 * GST_SECOND, TRUE}, + }; + TWCCPacket packets3[] = { + {7, 7 * GST_SECOND, FALSE}, + {8, 8 * GST_SECOND, TRUE}, }; -*/ - twcc_push_packets (recv_h, packets); + + twcc_push_packets (recv_h, packets0); + twcc_push_packets (recv_h, packets1); + twcc_push_packets (recv_h, packets2); + twcc_push_packets (recv_h, packets3); buf[0] = session_harness_produce_twcc (recv_h); buf[1] = session_harness_produce_twcc (recv_h); @@ -3960,16 +4401,16 @@ GST_START_TEST (test_twcc_recv_rtcp_reordered) gst_event_unref (gst_harness_pull_upstream_event (send_h->send_rtp_h)); event = gst_harness_pull_upstream_event (send_h->send_rtp_h); - twcc_verify_packets_to_event (&packets[0 * 2], event); + twcc_verify_packets_to_event (packets0, event); event = gst_harness_pull_upstream_event (send_h->send_rtp_h); - twcc_verify_packets_to_event (&packets[2 * 2], event); + twcc_verify_packets_to_event (packets2, event); event = gst_harness_pull_upstream_event (send_h->send_rtp_h); - twcc_verify_packets_to_event (&packets[1 * 2], event); + twcc_verify_packets_to_event (packets1, event); event = gst_harness_pull_upstream_event (send_h->send_rtp_h); - twcc_verify_packets_to_event (&packets[3 * 2], event); + twcc_verify_packets_to_event (packets3, event); session_harness_free (send_h); session_harness_free (recv_h); @@ -4000,7 +4441,7 @@ GST_START_TEST (test_twcc_send_and_recv) /* enable twcc */ session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); - session_harness_set_twcc_send_ext_id (h_send, TEST_TWCC_EXT_ID); + session_harness_add_twcc_caps_for_pt (h_send, TEST_BUF_PT); for (frame = 0; frame < num_frames; frame++) { GstBuffer *buf; @@ -4030,9 +4471,12 @@ GST_START_TEST (test_twcc_send_and_recv) /* sender receives the TWCC packet */ session_harness_recv_rtcp (h_send, buf); - if (frame > 0) - twcc_verify_stats (h_send, TEST_BUF_BPS, TEST_BUF_BPS, num_slices, - num_slices, 0.0f, 0); + if (frame > 0) { + GstStructure *twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 532800, 532800, num_slices + 1, + num_slices + 1, 0.0f, 0); + gst_structure_free (twcc_stats); + } } session_harness_free (h_send); @@ -4045,8 +4489,7 @@ GST_START_TEST (test_twcc_multiple_payloads_below_window) { SessionHarness *h_send = session_harness_new (); SessionHarness *h_recv = session_harness_new (); - - guint i; + GstStructure *twcc_stats; GstBuffer *buffers[] = { generate_twcc_send_buffer_full (0, FALSE, 0xabc, 98), @@ -4057,28 +4500,13 @@ GST_START_TEST (test_twcc_multiple_payloads_below_window) }; /* enable twcc */ - session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); - session_harness_set_twcc_send_ext_id (h_send, TEST_TWCC_EXT_ID); - - for (i = 0; i < G_N_ELEMENTS (buffers); i++) { - GstBuffer *buf = buffers[i]; - GstFlowReturn res; - - /* from payloder to rtpbin */ - res = session_harness_send_rtp (h_send, buf); - fail_unless_equals_int (GST_FLOW_OK, res); - - buf = session_harness_pull_send_rtp (h_send); - session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + session_harness_add_twcc_caps_for_pt (h_send, 98); + session_harness_add_twcc_caps_for_pt (h_send, 111); - /* buffer arrives at the receiver */ - res = session_harness_recv_rtp (h_recv, buf); - fail_unless_equals_int (GST_FLOW_OK, res); - } - - /* sender receives the TWCC packet from the receiver */ - session_harness_recv_rtcp (h_send, session_harness_produce_twcc (h_recv)); - twcc_verify_stats (h_send, 0, 0, 5, 5, 0.0f, GST_CLOCK_STIME_NONE); + twcc_send_recv_buffers (h_send, h_recv, buffers); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 0, 0, 0, 0, 0.0f, 0); + gst_structure_free (twcc_stats); session_harness_free (h_send); session_harness_free (h_recv); @@ -4288,6 +4716,263 @@ GST_START_TEST (test_twcc_feedback_old_seqnum) GST_END_TEST; +static guint +construct_initial_state_for_rtx (SessionHarness * h_send, + SessionHarness * h_recv) +{ + guint i; + guint window_size_ms = 300; + guint num_buffers = window_size_ms / TEST_BUF_MS + 1; + + session_harness_enable_rtx (h_send); + session_harness_enable_rtx (h_recv); + + /* send and recv enough packets to be over the stats window */ + for (i = 0; i < num_buffers; i++) { + GstFlowReturn ret; + GstBuffer *buf; + gboolean is_last = (i == num_buffers - 1); + + buf = generate_twcc_send_buffer (i, is_last); + ret = session_harness_send_rtp (h_send, buf); + fail_unless_equals_int (ret, GST_FLOW_OK); + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + + buf = session_harness_pull_send_rtp (h_send); + ret = session_harness_recv_rtp (h_recv, buf); + fail_unless_equals_int (ret, GST_FLOW_OK); + session_harness_advance_and_crank (h_recv, TEST_BUF_DURATION); + } + + /* produce a twcc feedback to process those packets */ + session_harness_recv_rtcp (h_send, session_harness_produce_twcc (h_recv)); + + return i; +} + +static void +fail_unless_twcc_stats_recovery (SessionHarness * h, gdouble recovery_pct) +{ + gdouble stats_recovery_pct; + GstStructure *twcc_stats; + + twcc_stats = session_harness_get_twcc_stats_full (h, + 300 * GST_MSECOND, 100 * GST_MSECOND); + + fail_unless (gst_structure_get (twcc_stats, + "recovery-pct", G_TYPE_DOUBLE, &stats_recovery_pct, NULL)); + fail_unless_equals_float (recovery_pct, stats_recovery_pct); + + gst_structure_free (twcc_stats); +} + +static void +send_recv_buffer (SessionHarness * h_send, SessionHarness * h_recv, + GstBuffer * buf, gboolean recv_buf) +{ + GstFlowReturn ret; + + ret = session_harness_send_rtp (h_send, buf); + fail_unless_equals_int64 (ret, GST_FLOW_OK); + + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + + if (recv_buf) { + ret = session_harness_recv_rtp (h_recv, + session_harness_pull_send_rtp (h_send)); + fail_unless_equals_int64 (ret, GST_FLOW_OK); + session_harness_advance_and_crank (h_recv, TEST_BUF_DURATION); + } else { + /* discard that buffer from the sender */ + gst_buffer_unref (session_harness_pull_send_rtp (h_send)); + } + +} + +static void +test_twcc_stats_rtx_recovery (gboolean rtx_arrive, gdouble recovery_pct) +{ + SessionHarness *h_send = session_harness_new (); + SessionHarness *h_recv = session_harness_new (); + guint i, next_seqnum; + + session_harness_add_twcc_caps_for_pt (h_send, TEST_BUF_PT); + session_harness_add_twcc_caps_for_pt (h_send, TEST_RTX_BUF_PT); + session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); + + next_seqnum = construct_initial_state_for_rtx (h_send, h_recv); + + for (i = 0; i < 3; i++) { + GstBuffer *buf; + GstBuffer *rtx_buf; + + buf = generate_twcc_send_buffer (next_seqnum++, FALSE); + rtx_buf = generate_rtx_buffer (i, buf); + + /* we send a buffer but receiver doesn't get it */ + send_recv_buffer (h_send, h_recv, buf, FALSE); + send_recv_buffer (h_send, h_recv, rtx_buf, rtx_arrive); + } + + /* push a last buffer with the marker bit to trigger the report */ + send_recv_buffer (h_send, h_recv, + generate_twcc_send_buffer (next_seqnum++, TRUE), TRUE); + + fail_unless_equals_int64 (GST_FLOW_OK, + session_harness_recv_rtcp (h_send, + session_harness_produce_twcc (h_recv))); + + fail_unless_twcc_stats_recovery (h_send, recovery_pct); + + session_harness_free (h_send); + session_harness_free (h_recv); +} + +GST_START_TEST (test_twcc_stats_rtx_recover_lost) +{ + test_twcc_stats_rtx_recovery (TRUE, 100.0); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_stats_no_rtx_no_recover) +{ + test_twcc_stats_rtx_recovery (FALSE, 0.0); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_feedback_max_sent_packets) +{ + /* test is very intense for valgrind or debug build */ + if (RUNNING_ON_VALGRIND || g_getenv ("EXECUTING_UNDER_DEBUG_BUILD")) + return; + + SessionHarness *h = session_harness_new (); + GstStructure *twcc_stats; + guint i; + + guint8 fci[] = { + 0x00, 0x00, /* base sequence number: 0 */ + 0xff, 0xff, /* packet status count: 65535 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ + 0x00, /* feedback packet count: 0 */ + 0x1f, 0xff, /* (9x) run-length with max length as not recv: 0 0 0 1 1 1 1 1 | 1 1 1 1 1 1 1 1 */ + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + }; + + session_harness_add_twcc_caps_for_pt (h, TEST_BUF_PT); + + /* send over 65536 packets */ + for (i = 0; i < 65536 + 65535; i++) { + session_harness_send_rtp (h, generate_twcc_send_buffer (i, FALSE)); + session_harness_advance_and_crank (h, TEST_BUF_DURATION); + } + + /* receive the feedback message and verify the packets in our window */ + session_harness_recv_rtcp (h, + generate_twcc_feedback_rtcp (fci, sizeof (fci))); + twcc_stats = session_harness_get_twcc_stats_full (h, + 1000 * GST_MSECOND, 40 * GST_MSECOND); + twcc_verify_stats (twcc_stats, 532800, 0, 51, 0, 100.0f, 0); + gst_structure_free (twcc_stats); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_non_twcc_pkts_does_not_mark_loss) +{ + SessionHarness *h_send = session_harness_new (); + SessionHarness *h_recv = session_harness_new (); + GstStructure *twcc_stats; + + /* *INDENT-OFF* */ + GstBuffer *buffers[] = { + generate_twcc_send_buffer_full (0, FALSE, 0xabc, 98), + generate_twcc_send_buffer_full (0, FALSE, 0xabc, 99), + generate_twcc_send_buffer_full (1, FALSE, 0xabc, 99), + generate_twcc_send_buffer_full (1, TRUE, 0xabc, 98), + generate_twcc_send_buffer_full (2, TRUE, 0xabc, 99), + }; + /* *INDENT-ON* */ + + session_harness_add_twcc_caps_for_pt (h_send, 99); + + twcc_send_recv_buffers (h_send, h_recv, buffers); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 0, 0, 0, 0, 0.0f, 0); + gst_structure_free (twcc_stats); + + session_harness_free (h_send); + session_harness_free (h_recv); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_non_twcc_pt_no_twcc_seqnum) +{ + SessionHarness *h; + GstBuffer *buf; + + h = session_harness_new (); + session_harness_add_twcc_caps_for_pt (h, 99); + + /* push a buffer with a pt marked for twcc */ + buf = generate_twcc_send_buffer_full (0, FALSE, 0xabc, 99); + fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); + + /* expect the twcc-seqnum to be written */ + buf = session_harness_pull_send_rtp (h); + fail_unless (read_twcc_seqnum (buf, TEST_TWCC_EXT_ID) == 0); + gst_buffer_unref (buf); + + /* push a buffer with a different pt */ + buf = generate_twcc_send_buffer_full (0, FALSE, 0xabc, 98); + fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); + + /* expect no twcc-seqnum */ + buf = session_harness_pull_send_rtp (h); + fail_unless (read_twcc_seqnum (buf, TEST_TWCC_EXT_ID) == -1); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_overwrites_exthdr_seqnum_if_present) +{ + SessionHarness *h; + GstBuffer *buf; + + h = session_harness_new (); + session_harness_add_twcc_caps_for_pt (h, TEST_BUF_PT); + + /* sending the ext-id in generate_test_buffer_full() will add the one-byte + header extension */ + buf = + generate_test_buffer_full (0, 0, 0, 0xabc, FALSE, TEST_BUF_PT, + TEST_TWCC_EXT_ID, 255); + fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); + + buf = session_harness_pull_send_rtp (h); + fail_unless (read_twcc_seqnum (buf, TEST_TWCC_EXT_ID) != 255); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + GST_START_TEST (test_twcc_run_length_max) { SessionHarness *h0 = session_harness_new (); @@ -4295,8 +4980,8 @@ GST_START_TEST (test_twcc_run_length_max) TWCCPacket packets[] = { /* *INDENT-OFF* */ - {0, 1000 * GST_USECOND, FALSE}, - {8205, 2000 * GST_USECOND, TRUE}, + { 0, 1000 * GST_USECOND, FALSE }, + { 8205, 2000 * GST_USECOND, TRUE }, /* *INDENT-ON* */ }; @@ -4315,7 +5000,7 @@ GST_START_TEST (test_twcc_run_length_max) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -4330,8 +5015,8 @@ GST_START_TEST (test_twcc_run_length_min) TWCCPacket packets[] = { /* *INDENT-OFF* */ - {0, 1000 * GST_USECOND, FALSE}, - {29, 2000 * GST_USECOND, TRUE}, + { 0, 1000 * GST_USECOND, FALSE }, + { 29, 2000 * GST_USECOND, TRUE }, /* *INDENT-ON* */ }; @@ -4350,7 +5035,7 @@ GST_START_TEST (test_twcc_run_length_min) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -4394,6 +5079,65 @@ GST_START_TEST (test_twcc_feedback_interval_new_internal_source) GST_END_TEST; +G_GNUC_NO_INSTRUMENT static void +_count_warns_log_func (G_GNUC_UNUSED GstDebugCategory * category, + GstDebugLevel level, + G_GNUC_UNUSED const gchar * file, + G_GNUC_UNUSED const gchar * function, + G_GNUC_UNUSED gint line, + G_GNUC_UNUSED GObject * object, + G_GNUC_UNUSED GstDebugMessage * message, gpointer user_data) +{ + if (level == GST_LEVEL_WARNING) { + guint *logged_warnings = user_data; + (*logged_warnings)++; + } +} + +GST_START_TEST (test_twcc_sent_packets_wrap) +{ + SessionHarness *h; + gint i; + guint logged_warnings = 0; + GstDebugLevel level = gst_debug_get_default_threshold (); + + h = session_harness_new (); + session_harness_add_twcc_caps_for_pt (h, TEST_BUF_PT); + + gst_debug_set_default_threshold (GST_LEVEL_WARNING); + gst_debug_add_log_function (_count_warns_log_func, &logged_warnings, NULL); + + for (i = 0; i < 65536; i++) { + GstBuffer *buf; + GstTxFeedbackMeta *meta; + GstClockTime ts; + gboolean marker; + + marker = (i == 65536 - 1); + buf = generate_twcc_send_buffer (i, marker); + ts = i * TEST_BUF_DURATION; + + fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); + + buf = session_harness_pull_send_rtp (h); + meta = gst_buffer_get_tx_feedback_meta (buf); + fail_unless (meta); + + gst_tx_feedback_meta_set_tx_time (meta, ts); + gst_buffer_unref (buf); + } + + /* demand no warnings have been logged, means we have should have set all + sent packets "send" timestamps */ + fail_unless_equals_uint64 (logged_warnings, 0); + + session_harness_free (h); + + gst_debug_set_default_threshold (level); +} + +GST_END_TEST; + static Suite * rtpsession_suite (void) { @@ -4457,12 +5201,18 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_twcc_huge_seqnum_gap); tcase_add_test (tc_chain, test_twcc_double_packets); tcase_add_test (tc_chain, test_twcc_duplicate_seqnums); + tcase_add_test (tc_chain, test_twcc_duplicate_previous_seqnums); + tcase_add_test (tc_chain, test_twcc_missing_packet_duplicates_last); tcase_add_test (tc_chain, test_twcc_multiple_markers); tcase_add_test (tc_chain, test_twcc_no_marker_and_gaps); tcase_add_test (tc_chain, test_twcc_bad_rtcp); tcase_add_test (tc_chain, test_twcc_delta_ts_rounding); tcase_add_test (tc_chain, test_twcc_double_gap); tcase_add_test (tc_chain, test_twcc_recv_packets_reordered); + tcase_add_test (tc_chain, test_twcc_recv_packets_reordered_and_lost); + tcase_add_test (tc_chain, + test_twcc_recv_packets_reordered_within_report_interval); + tcase_add_test (tc_chain, test_twcc_reordering_send_recv); tcase_add_test (tc_chain, test_twcc_recv_late_packet_fb_pkt_count_wrap); tcase_add_test (tc_chain, test_twcc_recv_rtcp_reordered); tcase_add_test (tc_chain, test_twcc_no_exthdr_in_buffer); @@ -4473,6 +5223,13 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_twcc_feedback_interval_new_internal_source); tcase_add_test (tc_chain, test_twcc_feedback_count_wrap); tcase_add_test (tc_chain, test_twcc_feedback_old_seqnum); + tcase_add_test (tc_chain, test_twcc_stats_rtx_recover_lost); + tcase_add_test (tc_chain, test_twcc_stats_no_rtx_no_recover); + tcase_add_test (tc_chain, test_twcc_feedback_max_sent_packets); + tcase_add_test (tc_chain, test_twcc_non_twcc_pkts_does_not_mark_loss); + tcase_add_test (tc_chain, test_twcc_non_twcc_pt_no_twcc_seqnum); + tcase_add_test (tc_chain, test_twcc_overwrites_exthdr_seqnum_if_present); + tcase_add_test (tc_chain, test_twcc_sent_packets_wrap); return s; } From 18de2409fec78091e092199ffb9bdbfb99fe2b78 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 16 Feb 2022 15:26:22 -0300 Subject: [PATCH 004/339] rtpfunnel: removing twcc functionality, rtptwcc is fully responsible now --- .../gst/rtpmanager/gstrtpfunnel.c | 93 -------- .../tests/check/elements/rtpfunnel.c | 214 +----------------- 2 files changed, 4 insertions(+), 303 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c index 841b55b00fb..5abc16b6b82 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c @@ -53,12 +53,6 @@ * RTP funnel can than make sure that this event hits the right encoder based * on the SSRC embedded in the event. * - * Another feature of the RTP funnel is that it will mux together TWCC - * (Transport-Wide Congestion Control) sequence-numbers. The point being that - * it should increment "transport-wide", meaning potentially several - * bundled streams. Note that not *all* streams being bundled needs to be - * affected by this. As an example Google WebRTC will use bundle with audio - * and video, but will only use TWCC sequence-numbers for the video-stream(s). * */ @@ -86,7 +80,6 @@ struct _GstRtpFunnelPad { GstPad pad; guint32 ssrc; - gboolean has_twcc; }; G_DEFINE_TYPE (GstRtpFunnelPad, gst_rtp_funnel_pad, GST_TYPE_PAD); @@ -129,9 +122,6 @@ struct _GstRtpFunnel /* The last pad data was chained on */ GstPad *current_pad; - guint twcc_pads; /* numer of sinkpads with negotiated twcc */ - GstRTPHeaderExtension *twcc_ext; - /* properties */ gint common_ts_offset; }; @@ -218,47 +208,6 @@ gst_rtp_funnel_forward_segment (GstRtpFunnel * funnel, GstPad * pad) return; } -static void -gst_rtp_funnel_set_twcc_seqnum (GstRtpFunnel * funnel, - GstPad * pad, GstBuffer ** buf) -{ - GstRtpFunnelPad *fpad = GST_RTP_FUNNEL_PAD_CAST (pad); - guint8 twcc_seq[2] = { 0, }; - GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; - guint ext_id = gst_rtp_header_extension_get_id (funnel->twcc_ext); - guint8 *existing; - guint size; - - if (!funnel->twcc_ext || !fpad->has_twcc) - return; - - *buf = gst_buffer_make_writable (*buf); - - gst_rtp_header_extension_write (funnel->twcc_ext, *buf, - GST_RTP_HEADER_EXTENSION_ONE_BYTE, *buf, twcc_seq, sizeof (twcc_seq)); - - if (!gst_rtp_buffer_map (*buf, GST_MAP_READWRITE, &rtp)) - goto map_failed; - - if (gst_rtp_buffer_get_extension_onebyte_header (&rtp, ext_id, - 0, (gpointer) & existing, &size)) { - if (size >= gst_rtp_header_extension_get_max_size (funnel->twcc_ext, *buf)) { - existing[0] = twcc_seq[0]; - existing[1] = twcc_seq[1]; - } - } - /* TODO: two-byte variant */ - - gst_rtp_buffer_unmap (&rtp); - - return; - -map_failed: - { - GST_ERROR ("failed to map buffer %p", *buf); - } -} - static GstFlowReturn gst_rtp_funnel_sink_chain_object (GstPad * pad, GstRtpFunnel * funnel, gboolean is_list, GstMiniObject * obj) @@ -276,7 +225,6 @@ gst_rtp_funnel_sink_chain_object (GstPad * pad, GstRtpFunnel * funnel, res = gst_pad_push_list (funnel->srcpad, GST_BUFFER_LIST_CAST (obj)); } else { GstBuffer *buf = GST_BUFFER_CAST (obj); - gst_rtp_funnel_set_twcc_seqnum (funnel, pad, &buf); res = gst_pad_push (funnel->srcpad, buf); } GST_PAD_STREAM_UNLOCK (funnel->srcpad); @@ -303,33 +251,6 @@ gst_rtp_funnel_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GST_MINI_OBJECT_CAST (buffer)); } -static void -gst_rtp_funnel_set_twcc_ext_id (GstRtpFunnel * funnel, guint8 twcc_ext_id) -{ - gchar *name; - guint current_ext_id; - - current_ext_id = gst_rtp_header_extension_get_id (funnel->twcc_ext); - g_object_set (funnel->twcc_ext, "n-streams", funnel->twcc_pads, NULL); - - if (current_ext_id == twcc_ext_id) - return; - - name = g_strdup_printf ("extmap-%u", twcc_ext_id); - - gst_caps_set_simple (funnel->srccaps, name, G_TYPE_STRING, - gst_rtp_header_extension_get_uri (funnel->twcc_ext), NULL); - - g_free (name); - - /* make sure we update the sticky with the new caps */ - funnel->send_sticky_events = TRUE; - - gst_rtp_header_extension_set_id (funnel->twcc_ext, twcc_ext_id); -} - -#define TWCC_EXTMAP_STR "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" - static gboolean gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -351,7 +272,6 @@ gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) GstCaps *caps; GstStructure *s; guint ssrc; - guint8 ext_id; GstCaps *rtpcaps = gst_caps_new_empty_simple (RTP_CAPS); gst_event_parse_caps (event, &caps); @@ -372,16 +292,6 @@ gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) g_hash_table_insert (funnel->ssrc_to_pad, GUINT_TO_POINTER (ssrc), pad); } - if (!funnel->twcc_ext) - funnel->twcc_ext = - gst_rtp_header_extension_create_from_uri (TWCC_EXTMAP_STR); - - ext_id = gst_rtp_get_extmap_id_for_attribute (s, TWCC_EXTMAP_STR); - if (ext_id > 0) { - fpad->has_twcc = TRUE; - funnel->twcc_pads++; - gst_rtp_funnel_set_twcc_ext_id (funnel, ext_id); - } GST_OBJECT_UNLOCK (funnel); forward = FALSE; @@ -617,9 +527,6 @@ gst_rtp_funnel_finalize (GObject * object) gst_caps_unref (funnel->srccaps); g_hash_table_destroy (funnel->ssrc_to_pad); - - gst_clear_object (&funnel->twcc_ext); - G_OBJECT_CLASS (parent_class)->finalize (object); } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c b/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c index 676127ca18d..d2a1a708087 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c @@ -128,7 +128,7 @@ GST_START_TEST (rtpfunnel_common_ts_offset) GST_END_TEST; static GstBuffer * -generate_test_buffer (guint seqnum, guint ssrc, guint8 twcc_ext_id) +generate_test_buffer (guint seqnum, guint ssrc) { GstBuffer *buf; GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; @@ -143,13 +143,6 @@ generate_test_buffer (guint seqnum, guint ssrc, guint8 twcc_ext_id) gst_rtp_buffer_set_timestamp (&rtp, seqnum * 160); gst_rtp_buffer_set_ssrc (&rtp, ssrc); - if (twcc_ext_id > 0) { - guint16 data; - GST_WRITE_UINT16_BE (&data, seqnum); - gst_rtp_buffer_add_extension_onebyte_header (&rtp, twcc_ext_id, - &data, sizeof (guint16)); - } - gst_rtp_buffer_unmap (&rtp); return buf; @@ -187,7 +180,7 @@ GST_START_TEST (rtpfunnel_custom_sticky) /* Send a buffer through first pad, expect the event to be the first one */ fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (500, 123, 0))); + gst_harness_push (h0, generate_test_buffer (500, 123))); for (;;) { event = gst_harness_pull_event (h); if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) @@ -205,7 +198,7 @@ GST_START_TEST (rtpfunnel_custom_sticky) /* Send a buffer through second pad, expect the event to be the second one */ fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h1, generate_test_buffer (500, 123, 0))); + gst_harness_push (h1, generate_test_buffer (500, 123))); for (;;) { event = gst_harness_pull_event (h); if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) @@ -224,7 +217,7 @@ GST_START_TEST (rtpfunnel_custom_sticky) * one */ fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (500, 123, 5))); + gst_harness_push (h0, generate_test_buffer (500, 123))); for (;;) { event = gst_harness_pull_event (h); if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) @@ -296,201 +289,6 @@ GST_START_TEST (rtpfunnel_stress) GST_END_TEST; -#define TWCC_EXTMAP_STR "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" - -#define BOGUS_EXTMAP_STR "http://www.ietf.org/id/bogus" - -GST_START_TEST (rtpfunnel_twcc_caps) -{ - GstHarness *h, *h0, *h1; - GstCaps *caps, *expected_caps; - - h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); - - /* request a sinkpad, set caps with twcc extmap */ - h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); - gst_harness_set_src_caps_str (h0, "application/x-rtp, " - "ssrc=(uint)123, extmap-5=" TWCC_EXTMAP_STR ""); - - /* request a second sinkpad, the extmap should not be - present in the caps when doing a caps-query downstream, - as we don't want to force upstream (typically a payloader) - to use the extension */ - h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); - caps = gst_pad_query_caps (GST_PAD_PEER (h1->srcpad), NULL); - expected_caps = gst_caps_new_empty_simple ("application/x-rtp"); - fail_unless (gst_caps_is_equal (expected_caps, caps)); - gst_caps_unref (caps); - gst_caps_unref (expected_caps); - - /* now try and set a different extmap for the same id on the other - * sinkpad, and verify this does not work */ - gst_harness_set_src_caps_str (h1, "application/x-rtp, " - "ssrc=(uint)456, extmap-5=" BOGUS_EXTMAP_STR ""); - caps = gst_pad_get_current_caps (GST_PAD_PEER (h1->srcpad)); - fail_unless (caps == NULL); - - /* ...but setting the right extmap (5) will work just fine */ - expected_caps = gst_caps_from_string ("application/x-rtp, " - "ssrc=(uint)456, extmap-5=" TWCC_EXTMAP_STR ""); - gst_harness_set_src_caps (h1, gst_caps_ref (expected_caps)); - caps = gst_pad_get_current_caps (GST_PAD_PEER (h1->srcpad)); - fail_unless (gst_caps_is_equal (expected_caps, caps)); - gst_caps_unref (caps); - gst_caps_unref (expected_caps); - - gst_harness_teardown (h); - gst_harness_teardown (h0); - gst_harness_teardown (h1); -} - -GST_END_TEST; - -static gint32 -get_twcc_seqnum (GstBuffer * buf, guint8 ext_id) -{ - GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; - gint32 val = -1; - gpointer data; - - gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp); - if (gst_rtp_buffer_get_extension_onebyte_header (&rtp, ext_id, - 0, &data, NULL)) { - val = GST_READ_UINT16_BE (data); - } - gst_rtp_buffer_unmap (&rtp); - - return val; -} - -static guint32 -get_ssrc (GstBuffer * buf) -{ - guint32 ssrc; - GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; - gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp); - ssrc = gst_rtp_buffer_get_ssrc (&rtp); - gst_rtp_buffer_unmap (&rtp); - return ssrc; -} - -GST_START_TEST (rtpfunnel_twcc_passthrough) -{ - GstHarness *h, *h0; - GstBuffer *buf; - guint16 offset = 65530; - guint packets = 40; - guint i; - - h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); - h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); - gst_harness_set_src_caps_str (h0, "application/x-rtp, " - "ssrc=(uint)123, extmap-5=" TWCC_EXTMAP_STR ""); - - /* push some packets with twcc seqnum */ - for (i = 0; i < packets; i++) { - guint16 seqnum = i + offset; - fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (seqnum, 123, 5))); - } - - /* and verify the seqnums stays unchanged through the funnel */ - for (i = 0; i < packets; i++) { - guint16 seqnum = i + offset; - buf = gst_harness_pull (h); - fail_unless_equals_int (seqnum, get_twcc_seqnum (buf, 5)); - gst_buffer_unref (buf); - } - - gst_harness_teardown (h); - gst_harness_teardown (h0); -} - -GST_END_TEST; - -GST_START_TEST (rtpfunnel_twcc_mux) -{ - GstHarness *h, *h0, *h1; - GstBuffer *buf; - - h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); - h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); - h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); - gst_harness_set_src_caps_str (h0, "application/x-rtp, " - "ssrc=(uint)123, extmap-5=" TWCC_EXTMAP_STR ""); - gst_harness_set_src_caps_str (h1, "application/x-rtp, " - "ssrc=(uint)456, extmap-5=" TWCC_EXTMAP_STR ""); - - /* push buffers on both pads with different twcc-seqnums on them (500 and 60000) */ - fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (500, 123, 5))); - fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h1, generate_test_buffer (60000, 321, 5))); - - /* verify they are muxed continuously (0 -> 1) */ - buf = gst_harness_pull (h); - fail_unless_equals_int (123, get_ssrc (buf)); - fail_unless_equals_int (0, get_twcc_seqnum (buf, 5)); - gst_buffer_unref (buf); - - buf = gst_harness_pull (h); - fail_unless_equals_int (321, get_ssrc (buf)); - fail_unless_equals_int (1, get_twcc_seqnum (buf, 5)); - gst_buffer_unref (buf); - - gst_harness_teardown (h); - gst_harness_teardown (h0); - gst_harness_teardown (h1); -} - -GST_END_TEST; - - -GST_START_TEST (rtpfunnel_twcc_passthrough_then_mux) -{ - GstHarness *h, *h0, *h1; - GstBuffer *buf; - guint offset0 = 500; - guint offset1 = 45678; - guint i; - - h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); - h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); - gst_harness_set_src_caps_str (h0, "application/x-rtp, " - "ssrc=(uint)123, extmap-5=" TWCC_EXTMAP_STR ""); - - /* push one packet with twcc seqnum 100 on pad0 */ - fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (offset0, 123, 5))); - - /* add pad1 to the funnel, also with twcc */ - h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); - gst_harness_set_src_caps_str (h1, "application/x-rtp, " - "ssrc=(uint)456, extmap-5=" TWCC_EXTMAP_STR ""); - - /* push one buffer on both pads, with pad1 starting at a different - offset */ - fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (offset0 + 1, 123, 5))); - fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h1, generate_test_buffer (offset1, 321, 5))); - - /* and verify the seqnums are continuous for all 3 packets, using - the inital offset from pad0 */ - for (i = 0; i < 3; i++) { - guint16 seqnum = i + offset0; - buf = gst_harness_pull (h); - fail_unless_equals_int (seqnum, get_twcc_seqnum (buf, 5)); - gst_buffer_unref (buf); - } - - gst_harness_teardown (h); - gst_harness_teardown (h0); - gst_harness_teardown (h1); -} - -GST_END_TEST; - static Suite * rtpfunnel_suite (void) { @@ -506,10 +304,6 @@ rtpfunnel_suite (void) tcase_add_test (tc_chain, rtpfunnel_stress); - tcase_add_test (tc_chain, rtpfunnel_twcc_caps); - tcase_add_test (tc_chain, rtpfunnel_twcc_passthrough); - tcase_add_test (tc_chain, rtpfunnel_twcc_mux); - tcase_add_test (tc_chain, rtpfunnel_twcc_passthrough_then_mux); return s; } From 3f1979973fd78139aaef190cf38a019947293bf3 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 15 May 2020 14:39:03 +0200 Subject: [PATCH 005/339] rtpfunnel: mark buffers as audio or video When bundling, the "media" caps goes away since this typically is combining audio and video together. By setting a flag on the buffers, much like what is done with RTX, downstream elements can make choices regarding these buffers without having to first map them and then have access to a payload-2-caps map. An example could be an element that paces the stream based on bitrate. Audio is usually already paced in its nature, and would only suffer from being held back based on the combined bitrate with video. In this case the element could simply check the flag to make the right decision. --- .../gst/rtpmanager/gstrtpfunnel.c | 22 ++++++++++++ .../tests/check/elements/rtpfunnel.c | 34 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c index 5abc16b6b82..e80c3ae0b08 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c @@ -80,6 +80,7 @@ struct _GstRtpFunnelPad { GstPad pad; guint32 ssrc; + GstRTPBufferFlags buffer_flag; }; G_DEFINE_TYPE (GstRtpFunnelPad, gst_rtp_funnel_pad, GST_TYPE_PAD); @@ -96,6 +97,24 @@ gst_rtp_funnel_pad_init (G_GNUC_UNUSED GstRtpFunnelPad * pad) { } +static void +gst_rtp_funnel_pad_set_buffer_flag (GstRtpFunnelPad * pad, GstBuffer * buf) +{ + if (pad->buffer_flag) + GST_BUFFER_FLAG_SET (buf, pad->buffer_flag); +} + +static void +gst_rtp_funnel_pad_set_media_type (GstRtpFunnelPad * pad, + const GstStructure * s) +{ + const gchar *media_type = gst_structure_get_string (s, "media"); + if (g_strcmp0 (media_type, "audio") == 0) + pad->buffer_flag = GST_RTP_BUFFER_FLAG_MEDIA_AUDIO; + else if (g_strcmp0 (media_type, "video") == 0) + pad->buffer_flag = GST_RTP_BUFFER_FLAG_MEDIA_VIDEO; +} + /**************** GstRTPFunnel ****************/ enum @@ -212,6 +231,7 @@ static GstFlowReturn gst_rtp_funnel_sink_chain_object (GstPad * pad, GstRtpFunnel * funnel, gboolean is_list, GstMiniObject * obj) { + GstRtpFunnelPad *fpad = GST_RTP_FUNNEL_PAD_CAST (pad); GstFlowReturn res; GST_DEBUG_OBJECT (pad, "received %" GST_PTR_FORMAT, obj); @@ -225,6 +245,7 @@ gst_rtp_funnel_sink_chain_object (GstPad * pad, GstRtpFunnel * funnel, res = gst_pad_push_list (funnel->srcpad, GST_BUFFER_LIST_CAST (obj)); } else { GstBuffer *buf = GST_BUFFER_CAST (obj); + gst_rtp_funnel_pad_set_buffer_flag (fpad, buf); res = gst_pad_push (funnel->srcpad, buf); } GST_PAD_STREAM_UNLOCK (funnel->srcpad); @@ -292,6 +313,7 @@ gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) g_hash_table_insert (funnel->ssrc_to_pad, GUINT_TO_POINTER (ssrc), pad); } + gst_rtp_funnel_pad_set_media_type (fpad, s); GST_OBJECT_UNLOCK (funnel); forward = FALSE; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c b/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c index d2a1a708087..1dcc86e976c 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c @@ -289,6 +289,39 @@ GST_START_TEST (rtpfunnel_stress) GST_END_TEST; +GST_START_TEST (rtpfunnel_mark_media_buffer_flags) +{ + GstHarness *h, *h0, *h1; + GstBuffer *buf; + + h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); + h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); + h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); + gst_harness_set_src_caps_str (h0, "application/x-rtp, " + "ssrc=(uint)123, media=(string)audio"); + gst_harness_set_src_caps_str (h1, "application/x-rtp, " + "ssrc=(uint)123, media=(string)video"); + + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h0, generate_test_buffer (0, 123))); + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h1, generate_test_buffer (0, 321))); + + buf = gst_harness_pull (h); + fail_unless (GST_BUFFER_FLAG_IS_SET (buf, GST_RTP_BUFFER_FLAG_MEDIA_AUDIO)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless (GST_BUFFER_FLAG_IS_SET (buf, GST_RTP_BUFFER_FLAG_MEDIA_VIDEO)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); + gst_harness_teardown (h0); + gst_harness_teardown (h1); +} + +GST_END_TEST; + static Suite * rtpfunnel_suite (void) { @@ -304,6 +337,7 @@ rtpfunnel_suite (void) tcase_add_test (tc_chain, rtpfunnel_stress); + tcase_add_test (tc_chain, rtpfunnel_mark_media_buffer_flags); return s; } From 3350a3eb2347ca27458f9dc3dc52f7ed0a93d544 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 8 Sep 2020 00:28:02 +0200 Subject: [PATCH 006/339] rtpfunnel: terminate upstream latency Scenario: You are receiving audio and video as RTP packets, and you want to mix the audio together with other audio, but you only want to "switch" the video-packets, meaning only repayloading. The network is terrible, so the jitterbuffer is configuring a high latency (200ms+) to deal with this. For the audio-stream the latency is terminated at the mixer/aggregator, where the 200ms ends up being the effective queue-size in front of the aggregator. For the video-stream however, there is no mixer, so the only place where this 200ms will be realized is in the sink (given that it has sync=TRUE, for sync=FALSE, lipsync now breaks horribly) There is a problem here however, and that is that there is crucial lipsync-information being calculated inside rtpbin based on the PTS on the buffers as they arrive inside gstrtpsession. What we end up with in this scenario is the equivalent of sending a stream that is timestamped 200ms apart, and then there is an enormous network-delay (not really, but this is what the sink "syncing" on PTS would appear as) that sort of brings the two streams together. Now there is code (that is default on) inside gstrtpsession to deal with this, in that any latency the pipeline has will be reported stored and added to the PTS before doing RTP lipsync calculations, however this will also be true for the stream that has its latency terminated by the aggregator, so there things will go wrong. And then there is bundle... All in all it seems like the best solution to this problem is to have the rtpfunnel "terminate" the upstream latency and add this straight to the PTS of the buffers passing through, making sure that rtpbin get the best possible conditions to keep lipsync in. --- .../gst/rtpmanager/gstrtpfunnel.c | 106 ++++++++++++++++++ .../tests/check/elements/rtpfunnel.c | 45 ++++++++ 2 files changed, 151 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c index e80c3ae0b08..84b76b2e295 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c @@ -81,6 +81,7 @@ struct _GstRtpFunnelPad GstPad pad; guint32 ssrc; GstRTPBufferFlags buffer_flag; + GstClockTime us_latency; }; G_DEFINE_TYPE (GstRtpFunnelPad, gst_rtp_funnel_pad, GST_TYPE_PAD); @@ -115,6 +116,35 @@ gst_rtp_funnel_pad_set_media_type (GstRtpFunnelPad * pad, pad->buffer_flag = GST_RTP_BUFFER_FLAG_MEDIA_VIDEO; } +static gboolean +gst_rtp_funnel_pad_query_latency (GstRtpFunnelPad * pad, + gboolean * _live, GstClockTime * _max) +{ + GstQuery *query = gst_query_new_latency (); + gboolean res; + + /* Ask peer for latency */ + res = gst_pad_peer_query (GST_PAD_CAST (pad), query); + + if (res) { + gboolean live; + GstClockTime min, max; + gst_query_parse_latency (query, &live, &min, &max); + + pad->us_latency = min; + GST_INFO_OBJECT (pad, "us_latency: %" GST_TIME_FORMAT, + GST_TIME_ARGS (pad->us_latency)); + + if (_live) + *_live = live; + if (_max) + *_max = max; + } + + gst_query_unref (query); + return res; +} + /**************** GstRTPFunnel ****************/ enum @@ -246,6 +276,7 @@ gst_rtp_funnel_sink_chain_object (GstPad * pad, GstRtpFunnel * funnel, } else { GstBuffer *buf = GST_BUFFER_CAST (obj); gst_rtp_funnel_pad_set_buffer_flag (fpad, buf); + GST_BUFFER_PTS (buf) += fpad->us_latency; res = gst_pad_push (funnel->srcpad, buf); } GST_PAD_STREAM_UNLOCK (funnel->srcpad); @@ -272,6 +303,61 @@ gst_rtp_funnel_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GST_MINI_OBJECT_CAST (buffer)); } +static gboolean +gst_rtp_funnel_query_latency (GstRtpFunnel * funnel, GstQuery * query) +{ + GstClockTime max = GST_CLOCK_TIME_NONE; + gboolean live = FALSE, done = FALSE; + GValue item = G_VALUE_INIT; + GstIterator *it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (funnel)); + + /* Take maximum of all latency values */ + while (!done) { + GstIteratorResult ires = gst_iterator_next (it, &item); + switch (ires) { + case GST_ITERATOR_DONE: + done = TRUE; + break; + case GST_ITERATOR_OK: + { + GstRtpFunnelPad *pad = g_value_get_object (&item); + gboolean live_cur; + GstClockTime max_cur; + + if (gst_rtp_funnel_pad_query_latency (pad, &live_cur, &max_cur)) { + /* max is the buffering-potential upstream, and we are only + ever as good as our weakest link here, so take the smallest + of the max values */ + max = MIN (max_cur, max); + /* if one branch is live, we are live */ + live = live || live_cur; + } + g_value_reset (&item); + break; + } + case GST_ITERATOR_RESYNC: + max = GST_CLOCK_TIME_NONE; + live = FALSE; + gst_iterator_resync (it); + break; + default: + done = TRUE; + break; + } + } + g_value_unset (&item); + gst_iterator_free (it); + + /* since we terminate upstream latency, we report 0 downstream */ + gst_query_set_latency (query, live, 0, max); + + /* store the results */ + GST_INFO_OBJECT (funnel, "Replying to latency query with: %" GST_PTR_FORMAT, + query); + + return TRUE; +} + static gboolean gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -434,6 +520,23 @@ gst_rtp_funnel_src_event (GstPad * pad, GstObject * parent, GstEvent * event) return ret; } +static gboolean +gst_rtp_funnel_src_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstRtpFunnel *funnel = GST_RTP_FUNNEL_CAST (parent); + gboolean res = FALSE; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_LATENCY: + res = gst_rtp_funnel_query_latency (funnel, query); + break; + default: + res = gst_pad_query_default (pad, parent, query); + break; + } + return res; +} + static GstPad * gst_rtp_funnel_request_new_pad (GstElement * element, GstPadTemplate * templ, const gchar * name, const GstCaps * caps) @@ -593,6 +696,9 @@ gst_rtp_funnel_init (GstRtpFunnel * funnel) gst_pad_use_fixed_caps (funnel->srcpad); gst_pad_set_event_function (funnel->srcpad, GST_DEBUG_FUNCPTR (gst_rtp_funnel_src_event)); + gst_pad_set_query_function (funnel->srcpad, + GST_DEBUG_FUNCPTR (gst_rtp_funnel_src_query)); + gst_element_add_pad (GST_ELEMENT (funnel), funnel->srcpad); funnel->send_sticky_events = TRUE; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c b/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c index 1dcc86e976c..fee82794673 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c @@ -322,6 +322,50 @@ GST_START_TEST (rtpfunnel_mark_media_buffer_flags) GST_END_TEST; +GST_START_TEST (rtpfunnel_terminate_latency) +{ + GstHarness *h, *h0, *h1; + GstBuffer *buf; + GstClockTime latency; + + h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); + h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); + h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); + gst_harness_set_src_caps_str (h0, "application/x-rtp, " + "ssrc=(uint)123, media=(string)audio"); + gst_harness_set_src_caps_str (h1, "application/x-rtp, " + "ssrc=(uint)321, media=(string)video"); + + gst_harness_set_upstream_latency (h0, 11 * GST_MSECOND); + gst_harness_set_upstream_latency (h1, 12 * GST_MSECOND); + + /* ask funnel about its latency */ + latency = gst_harness_query_latency (h); + /* verify it reports 0 */ + fail_unless_equals_uint64 (latency, 0); + + /* push buffers with PTS = 0 into the funnel */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h0, generate_test_buffer (0, 123))); + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h1, generate_test_buffer (0, 321))); + + /* and verify the upstream latency is now reflected as PTS on those + buffers */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (GST_BUFFER_PTS (buf), 11 * GST_MSECOND); + gst_buffer_unref (buf); + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (GST_BUFFER_PTS (buf), 12 * GST_MSECOND); + gst_buffer_unref (buf); + + gst_harness_teardown (h); + gst_harness_teardown (h0); + gst_harness_teardown (h1); +} + +GST_END_TEST; + static Suite * rtpfunnel_suite (void) { @@ -338,6 +382,7 @@ rtpfunnel_suite (void) tcase_add_test (tc_chain, rtpfunnel_stress); tcase_add_test (tc_chain, rtpfunnel_mark_media_buffer_flags); + tcase_add_test (tc_chain, rtpfunnel_terminate_latency); return s; } From b39319211b08ad138b6aace3f15495614df00880 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 21 Sep 2020 23:41:22 +0200 Subject: [PATCH 007/339] rtpfunnel: query us-latency if not configured --- subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c index 84b76b2e295..2a82d739a6a 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c @@ -82,6 +82,7 @@ struct _GstRtpFunnelPad guint32 ssrc; GstRTPBufferFlags buffer_flag; GstClockTime us_latency; + gboolean has_latency; }; G_DEFINE_TYPE (GstRtpFunnelPad, gst_rtp_funnel_pad, GST_TYPE_PAD); @@ -141,6 +142,8 @@ gst_rtp_funnel_pad_query_latency (GstRtpFunnelPad * pad, *_max = max; } + pad->has_latency = TRUE; + gst_query_unref (query); return res; } @@ -268,6 +271,10 @@ gst_rtp_funnel_sink_chain_object (GstPad * pad, GstRtpFunnel * funnel, GST_PAD_STREAM_LOCK (funnel->srcpad); + if (!fpad->has_latency) { + gst_rtp_funnel_pad_query_latency (fpad, NULL, NULL); + } + gst_rtp_funnel_send_sticky (funnel, pad); gst_rtp_funnel_forward_segment (funnel, pad); From 3a96c391f6bff4fbe695ac9a2eaf3f00f04a3b85 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 21 Sep 2020 23:41:43 +0200 Subject: [PATCH 008/339] rtpfunnel: query us-latency when receiving latency-changed event --- subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c index 2a82d739a6a..d422cf9bc68 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c @@ -412,6 +412,11 @@ gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) forward = FALSE; break; } + case GST_EVENT_LATENCY_CHANGED: + { + gst_rtp_funnel_pad_query_latency (fpad, NULL, NULL); + break; + } default: break; } From 1a3b2b08414ea365280311031d6ea538d14a8918 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 7 Apr 2022 10:54:46 +1000 Subject: [PATCH 009/339] rtpfunnel: revert back to pexip version [pexhack] Revisit when we do all the final changes around rtpsession and writing of TWCC seqnums. --- .../gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c index d422cf9bc68..3bf9d19d696 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c @@ -386,19 +386,16 @@ gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) GstCaps *caps; GstStructure *s; guint ssrc; - GstCaps *rtpcaps = gst_caps_new_empty_simple (RTP_CAPS); gst_event_parse_caps (event, &caps); GST_OBJECT_LOCK (funnel); - if (!gst_caps_can_intersect (rtpcaps, caps)) { + if (!gst_caps_can_intersect (funnel->srccaps, caps)) { GST_ERROR_OBJECT (funnel, "Can't intersect with caps %" GST_PTR_FORMAT, caps); g_assert_not_reached (); } - gst_caps_unref (rtpcaps); - s = gst_caps_get_structure (caps, 0); if (gst_structure_get_uint (s, "ssrc", &ssrc)) { fpad->ssrc = ssrc; @@ -441,17 +438,15 @@ gst_rtp_funnel_sink_query (GstPad * pad, GstObject * parent, GstQuery * query) { GstCaps *filter_caps; GstCaps *new_caps; - GstCaps *rtpcaps = gst_caps_new_empty_simple (RTP_CAPS); gst_query_parse_caps (query, &filter_caps); GST_OBJECT_LOCK (funnel); if (filter_caps) { - new_caps = gst_caps_intersect_full (rtpcaps, filter_caps, + new_caps = gst_caps_intersect_full (funnel->srccaps, filter_caps, GST_CAPS_INTERSECT_FIRST); - gst_caps_unref (rtpcaps); } else { - new_caps = rtpcaps; + new_caps = gst_caps_copy (funnel->srccaps); } GST_OBJECT_UNLOCK (funnel); From 5630d6312a8248fcba2fe528d930a9219d51d819 Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Mon, 26 May 2014 14:08:20 +0200 Subject: [PATCH 010/339] rtpjitterbuffer: Default do-lost to TRUE for rtpjitterbuffer [pexhack] Just for convenience --- .../gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 5c05386cc89..a9accdcae53 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -135,7 +135,7 @@ enum #define DEFAULT_DROP_ON_LATENCY FALSE #define DEFAULT_TS_OFFSET 0 #define DEFAULT_MAX_TS_OFFSET_ADJUSTMENT 0 -#define DEFAULT_DO_LOST FALSE +#define DEFAULT_DO_LOST TRUE #define DEFAULT_POST_DROP_MESSAGES FALSE #define DEFAULT_DROP_MESSAGES_INTERVAL_MS 200 #define DEFAULT_MODE RTP_JITTER_BUFFER_MODE_SLAVE From e9ced14ae45241164d745bd2d316472442866317 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sun, 22 May 2016 08:41:35 +1000 Subject: [PATCH 011/339] Revert "rtpjitterbuffer: Ensure to not take caps with the wrong pt for getting the clock-rate" This reverts commit 608b4ee53c43c3dcbfee167349771eeffd92e071. --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index a9accdcae53..b2e6c55f4cb 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -1506,12 +1506,11 @@ _get_cname_ssrc_mappings (GstRtpJitterBuffer * jitterbuffer, static gboolean gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, - GstCaps * caps, gint pt) + GstCaps * caps) { GstRtpJitterBufferPrivate *priv; GstStructure *caps_struct; guint val; - gint payload = -1; GstClockTime tval; const gchar *ts_refclk, *mediaclk; GstCaps *ts_meta_ref = NULL; @@ -1521,19 +1520,7 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, /* first parse the caps */ caps_struct = gst_caps_get_structure (caps, 0); - GST_DEBUG_OBJECT (jitterbuffer, "got caps %" GST_PTR_FORMAT, caps); - - if (gst_structure_get_int (caps_struct, "payload", &payload) && pt != -1 - && payload != pt) { - GST_ERROR_OBJECT (jitterbuffer, - "Got caps with wrong payload type (got %d, expected %d)", pt, payload); - return FALSE; - } - - if (payload != -1) { - GST_DEBUG_OBJECT (jitterbuffer, "Got payload type %d", payload); - priv->last_pt = payload; - } + GST_DEBUG_OBJECT (jitterbuffer, "got caps"); /* we need a clock-rate to convert the rtp timestamps to GStreamer time and to * measure the amount of data in the buffer */ @@ -1952,7 +1939,7 @@ queue_event (GstRtpJitterBuffer * jitterbuffer, GstEvent * event) GstCaps *caps; gst_event_parse_caps (event, &caps); - gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps, -1); + gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps); break; } case GST_EVENT_SEGMENT: @@ -2124,7 +2111,7 @@ gst_rtp_jitter_buffer_get_clock_rate (GstRtpJitterBuffer * jitterbuffer, if (!caps) goto no_caps; - res = gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps, pt); + res = gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps); gst_caps_unref (caps); if (G_UNLIKELY (!res)) @@ -3191,7 +3178,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, /* Try to get the clock-rate from the caps first if we can. If there are no * caps we must fire the signal to get the clock-rate. */ if ((caps = gst_pad_get_current_caps (pad))) { - gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps, pt); + gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps); gst_caps_unref (caps); } } From 321f597546128b898bcd89bfe6c5965f78747884 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 28 Oct 2020 01:10:18 +0100 Subject: [PATCH 012/339] rtpjitterbuffer: remove GFunc cast to silence GCC warnings --- .../tests/check/elements/rtpjitterbuffer.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c index 8984c350818..2e7ec1ccd5e 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c @@ -159,12 +159,19 @@ start_jitterbuffer (GstElement * jitterbuffer) return ret; } +/* wrapper for glib to get correct function signature */ +static void +gst_mini_object_unref_func (gpointer data, G_GNUC_UNUSED gpointer user_data) +{ + gst_mini_object_unref (data); +} + static void cleanup_jitterbuffer (GstElement * jitterbuffer) { GST_DEBUG ("cleanup_jitterbuffer"); - g_list_foreach (buffers, (GFunc) gst_mini_object_unref, NULL); + g_list_foreach (buffers, gst_mini_object_unref_func, NULL); g_list_free (buffers); buffers = NULL; From dbb940c2f6d4bd07bd7158c5345305e0d8e0b8cb Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Mon, 26 May 2014 14:07:39 +0200 Subject: [PATCH 013/339] rtpjitterbuffer: rtspsrc: Rename seqnum-base caps field to seqnum-offset RTP caps from payloaders use seqnum-offset, hence the jitterbuffer should use the same. Same for rtspsrc. --- .../gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c | 8 ++++---- subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index b2e6c55f4cb..fe074fd2fa8 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -1548,7 +1548,7 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, GST_DEBUG_OBJECT (jitterbuffer, "got clock-base %" G_GINT64_FORMAT, priv->clock_base); - if (gst_structure_get_uint (caps_struct, "seqnum-base", &val)) { + if (gst_structure_get_uint (caps_struct, "seqnum-offset", &val)) { /* first expected seqnum, only update when we didn't have a previous base. */ if (priv->next_in_seqnum == -1) priv->next_in_seqnum = val; @@ -1561,10 +1561,10 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, priv->seqnum_base = -1; } - GST_DEBUG_OBJECT (jitterbuffer, "got seqnum-base %d", priv->next_in_seqnum); + GST_DEBUG_OBJECT (jitterbuffer, "got seqnum-offset %d", priv->next_in_seqnum); - /* the start and stop times. The seqnum-base corresponds to the start time. We - * will keep track of the seqnums on the output and when we reach the one + /* the start and stop times. The seqnum-offset corresponds to the start time. + * We will keep track of the seqnums on the output and when we reach the one * corresponding to npt-stop, we emit the npt-stop-reached signal */ if (gst_structure_get_clock_time (caps_struct, "npt-start", &tval)) priv->npt_start = tval; diff --git a/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c b/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c index 864594d025a..a54f3e50537 100644 --- a/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c +++ b/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c @@ -5162,7 +5162,7 @@ gst_rtspsrc_configure_caps (GstRTSPSrc * src, GstSegment * segment, gst_caps_set_simple (caps, "clock-base", G_TYPE_UINT, (guint) stream->timebase, NULL); if (stream->seqbase != -1) - gst_caps_set_simple (caps, "seqnum-base", G_TYPE_UINT, + gst_caps_set_simple (caps, "seqnum-offset", G_TYPE_UINT, (guint) stream->seqbase, NULL); gst_caps_set_simple (caps, "npt-start", G_TYPE_UINT64, start, NULL); if (stop != -1) @@ -8847,7 +8847,7 @@ clear_rtp_base (GstRTSPSrc * src, GstRTSPStream * stream) item->caps = gst_caps_make_writable (item->caps); s = gst_caps_get_structure (item->caps, 0); - gst_structure_remove_fields (s, "clock-base", "seqnum-base", NULL); + gst_structure_remove_fields (s, "clock-base", "seqnum-offset", NULL); if (item->pt == stream->default_pt && stream->udpsrc[0]) g_object_set (stream->udpsrc[0], "caps", item->caps, NULL); } From 479f73da6817076578211876dd77daf125d19012 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 18 May 2018 13:50:41 +0200 Subject: [PATCH 014/339] rtpjitterbuffer: improve calculation of jitter and add more stats Given a constant source of jitter, the total-avg-jitter will be the "correct" amount of jitter experienced. Comparing this number to the running average for many long-running tests, having a slightly larger window seems to approximate the jitter much better. In addition we avoid weighting 0 too much when the initial calculations are done by ramping up the window and not counting the very first measurement. --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index fe074fd2fa8..82824a9098d 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -428,6 +428,11 @@ struct _GstRtpJitterBufferPrivate GstClockTime last_pts; guint64 last_rtptime; GstClockTime last_ntpnstime; + guint64 jitter_count; + GstClockTime max_jitter; + GstClockTime min_jitter; + guint64 jitter_sum; + GstClockTime total_avg_jitter; GstClockTime avg_jitter; /* for dropped packet messages */ @@ -1731,6 +1736,7 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer) priv->last_elapsed = 0; priv->ext_timestamp = -1; priv->avg_jitter = 0; + priv->min_jitter = G_MAXINT64; priv->last_dts = -1; priv->last_rtptime = -1; priv->last_ntpnstime = -1; @@ -2781,6 +2787,7 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, { gint32 rtpdiff; GstClockTimeDiff dtsdiff, rtpdiffns, diff; + gint factor; GstRtpJitterBufferPrivate *priv; priv = jitterbuffer->priv; @@ -2788,15 +2795,12 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, if (G_UNLIKELY (dts == GST_CLOCK_TIME_NONE) || priv->clock_rate <= 0) goto no_time; - if (priv->last_dts != -1) - dtsdiff = dts - priv->last_dts; - else - dtsdiff = 0; + /* we need at least two measurements to calculate jitter */ + if (priv->last_dts == -1 && priv->last_rtptime == -1) + goto done; - if (priv->last_rtptime != -1) - rtpdiff = rtptime - (guint32) priv->last_rtptime; - else - rtpdiff = 0; + dtsdiff = dts - priv->last_dts; + rtpdiff = rtptime - (guint32) priv->last_rtptime; /* Guess whether stream currently uses equidistant packet spacing. If we * often see identical timestamps it means the packets are not @@ -2807,9 +2811,6 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, priv->equidistant += 1; priv->equidistant = CLAMP (priv->equidistant, -7, 7); - priv->last_dts = dts; - priv->last_rtptime = rtptime; - if (rtpdiff > 0) rtpdiffns = gst_util_uint64_scale_int (rtpdiff, GST_SECOND, priv->clock_rate); @@ -2819,8 +2820,15 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, diff = ABS (dtsdiff - rtpdiffns); + priv->jitter_count++; + priv->jitter_sum += diff; + priv->total_avg_jitter = priv->jitter_sum / priv->jitter_count; + priv->max_jitter = MAX (priv->max_jitter, diff); + priv->min_jitter = MIN (priv->min_jitter, diff); + /* jitter is stored in nanoseconds */ - priv->avg_jitter = (diff + (15 * priv->avg_jitter)) >> 4; + factor = MIN (priv->jitter_count, 128); + priv->avg_jitter = (diff + ((factor - 1) * priv->avg_jitter)) / factor; GST_LOG_OBJECT (jitterbuffer, "dtsdiff %" GST_STIME_FORMAT " rtptime %" GST_STIME_FORMAT @@ -2828,6 +2836,10 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, GST_STIME_ARGS (dtsdiff), GST_STIME_ARGS (rtpdiffns), priv->clock_rate, GST_STIME_ARGS (diff), GST_TIME_ARGS (priv->avg_jitter)); +done: + priv->last_dts = dts; + priv->last_rtptime = rtptime; + return; /* ERRORS */ @@ -5383,6 +5395,10 @@ gst_rtp_jitter_buffer_create_stats (GstRtpJitterBuffer * jbuf) "num-late", G_TYPE_UINT64, priv->num_late, "num-duplicates", G_TYPE_UINT64, priv->num_duplicates, "avg-jitter", G_TYPE_UINT64, priv->avg_jitter, + "total-avg-jitter", G_TYPE_UINT64, priv->total_avg_jitter, + "min-jitter", G_TYPE_UINT64, priv->min_jitter, + "max-jitter", G_TYPE_UINT64, priv->max_jitter, + "latency-ms", G_TYPE_UINT, priv->latency_ms, "rtx-count", G_TYPE_UINT64, priv->num_rtx_requests, "rtx-success-count", G_TYPE_UINT64, priv->num_rtx_success, "rtx-per-packet", G_TYPE_DOUBLE, priv->avg_rtx_num, From cabd6dc26a8827112350a017cb9eba462be9d7a4 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 1 Oct 2018 14:09:58 +0200 Subject: [PATCH 015/339] rtpjitterbuffer: Optimize stats by caching quarks To avoid the GQuark global lock per entry. --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 82824a9098d..6648294fe31 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -595,6 +595,21 @@ static void update_rtx_stats (GstRtpJitterBuffer * jitterbuffer, static GstClockTime get_current_running_time (GstRtpJitterBuffer * jitterbuffer); +static GQuark quark_application_x_rtp_jitterbuffer_stats; +static GQuark quark_num_pushed; +static GQuark quark_num_lost; +static GQuark quark_num_late; +static GQuark quark_num_duplicates; +static GQuark quark_avg_jitter; +static GQuark quark_total_avg_jitter; +static GQuark quark_min_jitter; +static GQuark quark_max_jitter; +static GQuark quark_latency_ms; +static GQuark quark_rtx_count; +static GQuark quark_rtx_success_count; +static GQuark quark_rtx_per_packet; +static GQuark quark_rtx_rtt; + static void gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass) { @@ -604,6 +619,22 @@ gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass) gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; + quark_application_x_rtp_jitterbuffer_stats = + g_quark_from_static_string ("application/x-rtp-jitterbuffer-stats"); + quark_num_pushed = g_quark_from_static_string ("num-pushed"); + quark_num_lost = g_quark_from_static_string ("num-lost"); + quark_num_late = g_quark_from_static_string ("num-late"); + quark_num_duplicates = g_quark_from_static_string ("num-duplicates"); + quark_avg_jitter = g_quark_from_static_string ("avg-jitter"); + quark_total_avg_jitter = g_quark_from_static_string ("total-avg-jitter"); + quark_min_jitter = g_quark_from_static_string ("min-jitter"); + quark_max_jitter = g_quark_from_static_string ("max-jitter"); + quark_latency_ms = g_quark_from_static_string ("latency-ms"); + quark_rtx_count = g_quark_from_static_string ("rtx-count"); + quark_rtx_success_count = g_quark_from_static_string ("rtx-success-count"); + quark_rtx_per_packet = g_quark_from_static_string ("rtx-per-packet"); + quark_rtx_rtt = g_quark_from_static_string ("rtx-rtt"); + gobject_class->finalize = gst_rtp_jitter_buffer_finalize; gobject_class->set_property = gst_rtp_jitter_buffer_set_property; @@ -5389,20 +5420,20 @@ gst_rtp_jitter_buffer_create_stats (GstRtpJitterBuffer * jbuf) GstStructure *s; JBUF_LOCK (priv); - s = gst_structure_new ("application/x-rtp-jitterbuffer-stats", - "num-pushed", G_TYPE_UINT64, priv->num_pushed, - "num-lost", G_TYPE_UINT64, priv->num_lost, - "num-late", G_TYPE_UINT64, priv->num_late, - "num-duplicates", G_TYPE_UINT64, priv->num_duplicates, - "avg-jitter", G_TYPE_UINT64, priv->avg_jitter, - "total-avg-jitter", G_TYPE_UINT64, priv->total_avg_jitter, - "min-jitter", G_TYPE_UINT64, priv->min_jitter, - "max-jitter", G_TYPE_UINT64, priv->max_jitter, - "latency-ms", G_TYPE_UINT, priv->latency_ms, - "rtx-count", G_TYPE_UINT64, priv->num_rtx_requests, - "rtx-success-count", G_TYPE_UINT64, priv->num_rtx_success, - "rtx-per-packet", G_TYPE_DOUBLE, priv->avg_rtx_num, - "rtx-rtt", G_TYPE_UINT64, priv->avg_rtx_rtt, NULL); + s = gst_structure_new_id (quark_application_x_rtp_jitterbuffer_stats, + quark_num_pushed, G_TYPE_UINT64, priv->num_pushed, + quark_num_lost, G_TYPE_UINT64, priv->num_lost, + quark_num_late, G_TYPE_UINT64, priv->num_late, + quark_num_duplicates, G_TYPE_UINT64, priv->num_duplicates, + quark_avg_jitter, G_TYPE_UINT64, priv->avg_jitter, + quark_total_avg_jitter, G_TYPE_UINT64, priv->total_avg_jitter, + quark_min_jitter, G_TYPE_UINT64, priv->min_jitter, + quark_max_jitter, G_TYPE_UINT64, priv->max_jitter, + quark_latency_ms, G_TYPE_UINT, priv->latency_ms, + quark_rtx_count, G_TYPE_UINT64, priv->num_rtx_requests, + quark_rtx_success_count, G_TYPE_UINT64, priv->num_rtx_success, + quark_rtx_per_packet, G_TYPE_DOUBLE, priv->avg_rtx_num, + quark_rtx_rtt, G_TYPE_UINT64, priv->avg_rtx_rtt, NULL); JBUF_UNLOCK (priv); return s; From 1df48d8a1432af462e81c570af4dfb62d405bbfd Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 12 Jun 2019 10:52:21 +0200 Subject: [PATCH 016/339] rtpjitterbuffer: Send a latency-changed event when latency changes --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 4 ++ .../tests/check/elements/rtpjitterbuffer.c | 57 +++++++++++++++++-- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 6648294fe31..e019c6342b4 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -5099,6 +5099,10 @@ gst_rtp_jitter_buffer_set_property (GObject * object, gst_element_post_message (GST_ELEMENT_CAST (jitterbuffer), gst_message_new_latency (GST_OBJECT_CAST (jitterbuffer))); + + JBUF_LOCK (priv); + queue_event (jitterbuffer, gst_event_new_latency_changed ()); + JBUF_UNLOCK (priv); } break; } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c index 2e7ec1ccd5e..3a673c91eeb 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c @@ -712,6 +712,10 @@ construct_deterministic_initial_state (GstHarness * h, gint latency_ms) for (i = 0; i < 3; i++) gst_event_unref (gst_harness_pull_event (h)); + /* pull out the latency-changed event */ + if (gst_harness_events_in_queue (h) == 1) + gst_event_unref (gst_harness_pull_event (h)); + /* drop reconfigure event */ gst_event_unref (gst_harness_pull_upstream_event (h)); @@ -1249,6 +1253,9 @@ GST_START_TEST (test_all_packets_are_timestamped_zero) gst_harness_set_src_caps (h, generate_caps ()); g_object_set (h->element, "do-lost", TRUE, "latency", jb_latency_ms, NULL); + /* pull out the latency-changed event */ + gst_event_unref (gst_harness_pull_event (h)); + /* advance the clock with 10 seconds */ gst_harness_set_time (h, 10 * GST_SECOND); @@ -1368,6 +1375,9 @@ GST_START_TEST (test_loss_equidistant_spacing_with_parameter_packets) gst_harness_set_src_caps (h, generate_caps ()); g_object_set (h->element, "do-lost", TRUE, "latency", latency_ms, NULL); + /* pull out the latency-changed event */ + gst_event_unref (gst_harness_pull_event (h)); + /* drop stream-start, caps, segment */ for (i = 0; i < 3; i++) gst_event_unref (gst_harness_pull_event (h)); @@ -2470,8 +2480,8 @@ start_test_rtx_large_packet_spacing (GstHarness * h, gst_buffer_unref (gst_harness_pull (h)); } - /* drop GstEventStreamStart & GstEventCaps & GstEventSegment */ - for (i = 0; i < 3; i++) + /* drop GstEventStreamStart & GstEventCaps & GstEventSegment & Latency */ + for (i = 0; i < 4; i++) gst_event_unref (gst_harness_pull_event (h)); /* drop reconfigure event */ gst_event_unref (gst_harness_pull_upstream_event (h)); @@ -2606,8 +2616,8 @@ GST_START_TEST (test_rtx_large_packet_spacing_does_not_reset_jitterbuffer) gst_buffer_unref (buffer); } - /* drop GstEventStreamStart & GstEventCaps & GstEventSegment */ - for (i = 0; i < 3; i++) + /* drop GstEventStreamStart & GstEventCaps & GstEventSegment & Latency */ + for (i = 0; i < 4; i++) gst_event_unref (gst_harness_pull_event (h)); /* drop reconfigure event */ gst_event_unref (gst_harness_pull_upstream_event (h)); @@ -2931,6 +2941,9 @@ GST_START_TEST (test_considered_lost_packet_in_large_gap_arrives) testclock = gst_harness_get_testclock (h); g_object_set (h->element, "do-lost", TRUE, "latency", jb_latency_ms, NULL); + /* pull out the latency-changed event */ + gst_event_unref (gst_harness_pull_event (h)); + /* first push buffer 0 */ fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, generate_test_buffer_full (0 * TEST_BUF_DURATION, @@ -2980,6 +2993,40 @@ GST_START_TEST (test_considered_lost_packet_in_large_gap_arrives) GST_END_TEST; +GST_START_TEST (test_latency_changed_event) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + GstEvent *event; + + fail_unless_equals_int (0, gst_harness_events_received (h)); + + /* changing the latency should cause a downstream latency-changed event */ + g_object_set (h->element, "latency", 20, NULL); + + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_LATENCY_CHANGED, GST_EVENT_TYPE (event)); + gst_event_unref (event); + + fail_unless_equals_int (1, gst_harness_events_received (h)); + + /* same latency does not send the event again */ + g_object_set (h->element, "latency", 20, NULL); + fail_unless_equals_int (1, gst_harness_events_received (h)); + + /* and changing again causes another one */ + g_object_set (h->element, "latency", 40, NULL); + + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_LATENCY_CHANGED, GST_EVENT_TYPE (event)); + gst_event_unref (event); + + fail_unless_equals_int (2, gst_harness_events_received (h)); + + gst_harness_teardown (h); +} + +GST_END_TEST; + GST_START_TEST (test_performance) { GstHarness *h = @@ -3703,6 +3750,8 @@ rtpjitterbuffer_suite (void) test_considered_lost_packet_in_large_gap_arrives, 0, G_N_ELEMENTS (test_considered_lost_packet_in_large_gap_arrives_input)); + tcase_add_test (tc_chain, test_latency_changed_event); + tcase_add_test (tc_chain, test_performance); tcase_add_test (tc_chain, test_drop_messages_too_late); From 38de0e158d6a71140797c5a967584ed8569b5a82 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 16 Apr 2020 22:44:57 +0200 Subject: [PATCH 017/339] rtpjitterbuffer: don't calculate packet-rate for jumping seqnums As demonstrated by the test, you can end up in situations where your packet-rate gets insanly high, and you produce thousands of RTX-timers /lost events, when the big-gap logic is there to prevent exactly that. --- .../gst/rtpmanager/rtpstats.c | 6 ++-- .../tests/check/elements/rtpjitterbuffer.c | 30 +++++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c index 6c4662663e2..6b172034fb2 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c @@ -55,10 +55,10 @@ gst_rtp_packet_rate_ctx_update (RTPPacketRateCtx * ctx, guint16 seqnum, } diff_seqnum = gst_rtp_buffer_compare_seqnum (ctx->last_seqnum, seqnum); - /* Ignore seqnums that are over 15,000 away from the latest one, it's close - * to 2^14 but far enough to avoid any risk of computing error. + /* Ignore seqnums that are not strictly sequential from the last one. + That way we don't taint our estimate with "bad" packets */ - if (diff_seqnum > 15000) + if (diff_seqnum > 1) goto done_but_save; /* Ignore any packet that is in the past, we're only interested in newer diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c index 3a673c91eeb..8343f4fb298 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c @@ -2827,26 +2827,43 @@ GST_START_TEST (test_deadline_ts_offset_overflow) GST_END_TEST; +typedef struct +{ + GstClockTime delta_dts; + guint delta_seqnum; + guint delta_rtptime; +} BigGapCtx; + +/* TEST_RTP_TS_DURATION = 160 */ +static BigGapCtx big_gap_testdata[] = { + {0, 20000, 20000 * 160}, + {0, 10000, 10000 * 160}, + {0, 10000, 0 * 160}, + {0, 3000, 3000 * 160}, +}; + GST_START_TEST (test_big_gap_seqnum) { GstHarness *h = gst_harness_new ("rtpjitterbuffer"); const gint num_consecutive = 5; - const guint gap = 20000; gint i; guint seqnum_org; GstClockTime dts_base; guint seqnum_base; guint32 rtpts_base; GstClockTime expected_ts; + BigGapCtx *ctx = &big_gap_testdata[__i__]; g_object_set (h->element, "do-lost", TRUE, "do-retransmission", TRUE, NULL); seqnum_org = construct_deterministic_initial_state (h, 100); - /* a sudden jump in sequence-numbers (and rtptime), but packets keep arriving - at the same pace */ dts_base = seqnum_org * TEST_BUF_DURATION; - seqnum_base = seqnum_org + gap; - rtpts_base = seqnum_base * TEST_RTP_TS_DURATION; + seqnum_base = seqnum_org; + rtpts_base = seqnum_org * TEST_RTP_TS_DURATION; + + dts_base += ctx->delta_dts; + seqnum_base += ctx->delta_seqnum; + rtpts_base += ctx->delta_rtptime; for (i = 0; i < num_consecutive; i++) { fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, @@ -3742,7 +3759,8 @@ rtpjitterbuffer_suite (void) tcase_add_test (tc_chain, test_deadline_ts_offset); tcase_add_test (tc_chain, test_deadline_ts_offset_overflow); - tcase_add_test (tc_chain, test_big_gap_seqnum); + tcase_add_loop_test (tc_chain, test_big_gap_seqnum, 0, + G_N_ELEMENTS (big_gap_testdata)); tcase_add_test (tc_chain, test_big_gap_arrival_time); tcase_add_test (tc_chain, test_fill_queue); From f6df015beec7aea36f288cbae0b6b3c4b982c199 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 31 Mar 2020 14:06:28 +0200 Subject: [PATCH 018/339] rtpjitterbuffer: add failing test that reproduces a stall So what happens here is that at some point we receive packet #10508, which triggers the big gap logic, placing that buffer inside the gap_packets GList: gst_rtp_jitter_buffer_chain: expected #10529, got #10508, gap of -21 handle_big_gap_buffer: buffer too old -21 < -20, first one - waiting Then packet #10550 arrives: gst_rtp_jitter_buffer_chain: expected #10530, got #10550, gap of 20 This causes too many timers to be pending, causing a reset: gst_rtp_jitter_buffer_chain: 182 pending timers > 180 - resetting gst_rtp_jitter_buffer_reset:< flush and reset jitterbuffer Inside the reset() code, the next_seqnum is set to the first buffer in gap_packets, #10508: gst_rtp_jitter_buffer_reset: setting next_seqnum to #10508 And reset() calls chain() with all its gap_packets, pushing it out: gst_rtp_jitter_buffer_chain: Received packet #10508 at time 123:45:05.364650711, discont 0, rtx 1 However, #10508 is actually an RTX packet, so it gets dropped: gst_rtp_jitter_buffer_chain: Unsolicited RTX packet #10508 detected, dropping Next #10550 is pushed from reset(): gst_rtp_jitter_buffer_chain: Received packet #10550 at time 123:45:05.516136868, discont 0, rtx 0 But because next_seqnum is set to #10508, we get this: handle_next_buffer: Sequence number GAP detected: expected 10508 instead of 10550 (42 missing) So now the jitterbuffer is completely stalled, all buffer arriving now will just queue up until it sees the promised #10508, which will be ~65000 packets from now, and we will trigger the "full" logic before then, causing a deadlock. --- .../gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index e019c6342b4..c44a965f108 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -3023,6 +3023,9 @@ gst_rtp_jitter_buffer_reset (GstRtpJitterBuffer * jitterbuffer, priv->next_seqnum = seqnum; } + GST_DEBUG_OBJECT (jitterbuffer, "setting next_seqnum to #%u", + priv->next_seqnum); + priv->last_in_pts = -1; priv->next_in_seqnum = -1; From 63a39c74d535acb786005d699226c392797f997f Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sat, 4 Apr 2020 01:32:03 +0200 Subject: [PATCH 019/339] rtpjitterbuffer: remove reset-on-many-timers Too dangerous. --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index c44a965f108..32410e401c7 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -3341,21 +3341,6 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, GST_DEBUG_OBJECT (jitterbuffer, "expected #%d, got #%d, gap of %d", expected, seqnum, gap); - if (G_UNLIKELY (gap > 0 && - rtp_timer_queue_length (priv->timers) >= max_dropout)) { - /* If we have timers for more than RTP_MAX_DROPOUT packets - * pending this means that we have a huge gap overall. We can - * reset the jitterbuffer at this point because there's - * just too much data missing to be able to do anything - * sensible with the past data. Just try again from the - * next packet */ - GST_WARNING_OBJECT (jitterbuffer, "%d pending timers > %d - resetting", - rtp_timer_queue_length (priv->timers), max_dropout); - g_queue_insert_sorted (&priv->gap_packets, buffer, - (GCompareDataFunc) compare_buffer_seqnum, NULL); - return gst_rtp_jitter_buffer_reset (jitterbuffer, pad, parent, seqnum); - } - /* Special handling of large gaps */ if (!is_rtx && ((gap != -1 && gap < -max_misorder) || (gap >= max_dropout))) { gboolean reset = handle_big_gap_buffer (jitterbuffer, buffer, pt, seqnum, From eaf3bbb6d56767a05078eb0c9eb7a7257826eccd Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 31 May 2021 14:07:38 +0200 Subject: [PATCH 020/339] rtpjitterbuffer: fixing ulpfec packets used in pts and skew calculations. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Håvard Graff --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 44 ++- .../gst/rtpmanager/rtpjitterbuffer.c | 31 ++- .../tests/check/elements/rtpjitterbuffer.c | 251 ++++++++++++++++-- 3 files changed, 279 insertions(+), 47 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 32410e401c7..4f75df2c84b 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -286,6 +286,21 @@ enum #define GST_BUFFER_IS_RETRANSMISSION(buffer) \ GST_BUFFER_FLAG_IS_SET (buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION) +#define GST_BUFFER_IS_ULPFEC(buffer) \ + GST_BUFFER_FLAG_IS_SET (buffer, GST_RTP_BUFFER_FLAG_ULPFEC) + +#if !GLIB_CHECK_VERSION(2, 60, 0) +#define g_queue_clear_full queue_clear_full +static void +queue_clear_full (GQueue * queue, GDestroyNotify free_func) +{ + gpointer data; + + while ((data = g_queue_pop_head (queue)) != NULL) + free_func (data); +} +#endif + struct _GstRtpJitterBufferPrivate { GstPad *sinkpad, *srcpad; @@ -3153,6 +3168,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, gint32 packet_rate, max_dropout, max_misorder; RtpTimer *timer = NULL; gboolean is_rtx; + gboolean is_ulpfec; jitterbuffer = GST_RTP_JITTER_BUFFER_CAST (parent); @@ -3169,6 +3185,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, gst_rtp_buffer_unmap (&rtp); is_rtx = GST_BUFFER_IS_RETRANSMISSION (buffer); + is_ulpfec = GST_BUFFER_IS_ULPFEC (buffer); now = get_current_running_time (jitterbuffer); /* make sure we have PTS and DTS set */ @@ -3202,9 +3219,9 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, GST_DEBUG_OBJECT (jitterbuffer, "Received packet #%d at time %" GST_TIME_FORMAT - ", discont %d, rtx %d, inband NTP time %" GST_TIME_FORMAT, seqnum, - GST_TIME_ARGS (dts), GST_BUFFER_IS_DISCONT (buffer), is_rtx, - GST_TIME_ARGS (inband_ntp_time)); + ", discont %d, rtx %d ulpfec %d, inband NTP time %" GST_TIME_FORMAT, + seqnum, GST_TIME_ARGS (dts), GST_BUFFER_IS_DISCONT (buffer), is_rtx, + is_ulpfec, GST_TIME_ARGS (inband_ntp_time)); JBUF_LOCK_CHECK (priv, out_flushing); @@ -3255,7 +3272,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, if (G_UNLIKELY (priv->eos)) goto have_eos; - if (!is_rtx) + if (!is_rtx && !is_ulpfec) calculate_jitter (jitterbuffer, dts, rtptime); if (priv->seqnum_base != -1) { @@ -3280,7 +3297,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, expected = priv->next_in_seqnum; /* don't update packet-rate based on RTX, as those arrive highly unregularly */ - if (!is_rtx) { + if (!is_rtx && !is_ulpfec) { packet_rate = gst_rtp_packet_rate_ctx_update (&priv->packet_rate_ctx, seqnum, rtptime); GST_TRACE_OBJECT (jitterbuffer, "updated packet_rate: %d", packet_rate); @@ -3342,7 +3359,8 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, expected, seqnum, gap); /* Special handling of large gaps */ - if (!is_rtx && ((gap != -1 && gap < -max_misorder) || (gap >= max_dropout))) { + if (!is_rtx && !is_ulpfec && ((gap != -1 && gap < -max_misorder) + || (gap >= max_dropout))) { gboolean reset = handle_big_gap_buffer (jitterbuffer, buffer, pt, seqnum, gap, max_dropout, max_misorder); if (reset) { @@ -3364,11 +3382,15 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, pts = rtp_jitter_buffer_calculate_pts (priv->jbuf, dts, estimated_dts, rtptime, gst_element_get_base_time (GST_ELEMENT_CAST (jitterbuffer)), - gap, is_rtx, &ntp_time); + gap, is_rtx || is_ulpfec, &ntp_time); if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (pts))) { - /* A valid timestamp cannot be calculated, discard packet */ - goto discard_invalid; + if (is_ulpfec) { + pts = priv->ips_rtptime; + } else { + /* A valid timestamp cannot be calculated, discard packet */ + goto discard_invalid; + } } if (G_LIKELY (gap == 0)) { @@ -3664,8 +3686,8 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, discard_invalid: { GST_DEBUG_OBJECT (jitterbuffer, - "cannot calculate a valid pts for #%d (rtx: %d), discard", - seqnum, is_rtx); + "cannot calculate a valid pts for #%d (rtx: %d ulpfec: %d), discard", + seqnum, is_rtx, is_ulpfec); gst_buffer_unref (buffer); goto finished; } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpjitterbuffer.c index 873fcbe364f..2697f35ab08 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpjitterbuffer.c @@ -530,7 +530,7 @@ update_buffer_level (RTPJitterBuffer * jbuf, gint * percent) */ static GstClockTime calculate_skew (RTPJitterBuffer * jbuf, guint64 ext_rtptime, - GstClockTime gstrtptime, GstClockTime time, gint gap, gboolean is_rtx) + GstClockTime gstrtptime, GstClockTime time, gint gap, gboolean is_redundant) { guint64 send_diff, recv_diff; gint64 delta; @@ -544,7 +544,7 @@ calculate_skew (RTPJitterBuffer * jbuf, guint64 ext_rtptime, /* we don't have an arrival timestamp so we can't do skew detection. we * should still apply a timestamp based on RTP timestamp and base_time */ - if (time == -1 || jbuf->base_time == -1 || is_rtx) + if (time == -1 || jbuf->base_time == -1 || is_redundant) goto no_skew; /* elapsed time at receiver, includes the jitter */ @@ -700,7 +700,7 @@ queue_do_insert (RTPJitterBuffer * jbuf, GList * list, GList * item) GstClockTime rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, gboolean estimated_dts, guint32 rtptime, GstClockTime base_time, - gint gap, gboolean is_rtx, GstClockTime * p_ntp_time) + gint gap, gboolean is_redundant, GstClockTime * p_ntp_time) { guint64 ext_rtptime; GstClockTime gstrtptime, pts; @@ -723,13 +723,13 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, ext_rtptime = gst_rtp_buffer_ext_timestamp (&ext_rtptime, rtptime); if (ext_rtptime > jbuf->last_rtptime + 3 * jbuf->clock_rate || ext_rtptime + 3 * jbuf->clock_rate < jbuf->last_rtptime) { - if (!is_rtx) { + if (!is_redundant) { /* reset even if we don't have valid incoming time; * still better than producing possibly very bogus output timestamp */ GST_WARNING ("rtp delta too big, reset skew"); rtp_jitter_buffer_reset_skew (jbuf); } else { - GST_WARNING ("rtp delta too big: ignore rtx packet"); + GST_WARNING ("rtp delta too big: ignore redundant packet"); media_clock = NULL; pipeline_clock = NULL; pts = GST_CLOCK_TIME_NONE; @@ -760,14 +760,14 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, if (G_LIKELY (jbuf->base_rtptime != -1)) { /* check elapsed time in RTP units */ if (gstrtptime < jbuf->base_rtptime) { - if (!is_rtx) { + if (!is_redundant) { /* elapsed time at sender, timestamps can go backwards and thus be * smaller than our base time, schedule to take a new base time in * that case. */ GST_WARNING ("backward timestamps at server, schedule resync"); jbuf->need_resync = TRUE; } else { - GST_WARNING ("backward timestamps: ignore rtx packet"); + GST_DEBUG ("backward timestamps: ignore redundant packet"); pts = GST_CLOCK_TIME_NONE; goto done; } @@ -800,8 +800,8 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, /* need resync, lock on to time and gstrtptime if we can, otherwise we * do with the previous values */ if (G_UNLIKELY (jbuf->need_resync && dts != -1)) { - if (is_rtx) { - GST_DEBUG ("not resyncing on rtx packet, discard"); + if (is_redundant) { + GST_DEBUG ("not resyncing on redundant packet, discard"); pts = GST_CLOCK_TIME_NONE; goto done; } @@ -1004,7 +1004,8 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, /* do skew calculation by measuring the difference between rtptime and the * receive dts, this function will return the skew corrected rtptime. */ - pts = calculate_skew (jbuf, ext_rtptime, gstrtptime, dts, gap, is_rtx); + pts = + calculate_skew (jbuf, ext_rtptime, gstrtptime, dts, gap, is_redundant); } /* check if timestamps are not going backwards, we can only check this if we @@ -1027,12 +1028,18 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, } if (gap == 0 && dts != -1 && pts + jbuf->delay < dts) { + if (is_redundant) { + GST_DEBUG ("avoiding reset on redundant packet, discard"); + pts = GST_CLOCK_TIME_NONE; + goto done; + } + /* if we are going to produce a timestamp that is later than the input * timestamp, we need to reset the jitterbuffer. Likely the server paused * temporarily */ GST_DEBUG ("out %" GST_TIME_FORMAT " + %" G_GUINT64_FORMAT " < time %" - GST_TIME_FORMAT ", reset jitterbuffer and discard", GST_TIME_ARGS (pts), - jbuf->delay, GST_TIME_ARGS (dts)); + GST_TIME_FORMAT ", reset jitterbuffer and discard", + GST_TIME_ARGS (pts), jbuf->delay, GST_TIME_ARGS (dts)); rtp_jitter_buffer_reset_skew (jbuf); rtp_jitter_buffer_resync (jbuf, dts, gstrtptime, ext_rtptime, TRUE); pts = dts; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c index 8343f4fb298..707e30e41cd 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c @@ -657,9 +657,10 @@ verify_jb_stats (GstElement * jb, GstStructure * expected) } static guint -construct_deterministic_initial_state (GstHarness * h, gint latency_ms) +construct_deterministic_initial_state_full (GstHarness * h, + guint start_seqnum, gint latency_ms, GstCaps * caps) { - guint next_seqnum = latency_ms / TEST_BUF_MS + 1; + guint next_seqnum = start_seqnum + (latency_ms / TEST_BUF_MS + 1); guint seqnum; gint i; @@ -688,7 +689,7 @@ construct_deterministic_initial_state (GstHarness * h, gint latency_ms) */ /* Packet 0 arrives at time 0ms, Packet 5 arrives at time 100ms */ - for (seqnum = 0; seqnum < next_seqnum; seqnum++) { + for (seqnum = start_seqnum; seqnum < next_seqnum; seqnum++) { push_test_buffer (h, seqnum); gst_harness_wait_for_clock_id_waits (h, 1, 60); } @@ -699,8 +700,9 @@ construct_deterministic_initial_state (GstHarness * h, gint latency_ms) */ gst_harness_crank_single_clock_wait (h); fail_unless_equals_int64 (latency_ms * GST_MSECOND, - gst_clock_get_time (GST_ELEMENT_CLOCK (h->element))); - for (seqnum = 0; seqnum < next_seqnum; seqnum++) { + gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)) - + (start_seqnum * TEST_BUF_MS * GST_MSECOND)); + for (seqnum = start_seqnum; seqnum < next_seqnum; seqnum++) { GstBuffer *buf = gst_harness_pull (h); fail_unless_equals_uint64 (seqnum * TEST_BUF_DURATION, GST_BUFFER_PTS (buf)); @@ -791,6 +793,13 @@ setup_rtcp_pads (GstElement * jitterbuffer) return rtcp_fxsrc_pad; } +static guint +construct_deterministic_initial_state (GstHarness * h, gint latency_ms) +{ + return construct_deterministic_initial_state_full (h, 0, latency_ms, + generate_caps ()); +} + GST_START_TEST (test_lost_event) { GstHarness *h = gst_harness_new ("rtpjitterbuffer"); @@ -3115,39 +3124,34 @@ typedef struct { gint64 dts_skew; gint16 seqnum_skew; -} RtxSkewCtx; +} JitterBufferSkewCtx; -static const RtxSkewCtx rtx_does_not_affect_pts_calculation_input[] = { - {0, 0}, - {20 * GST_MSECOND, -100}, - {20 * GST_MSECOND, 100}, - {-10 * GST_MSECOND, 1}, - {100 * GST_MSECOND, 0}, -}; - -GST_START_TEST (test_rtx_does_not_affect_pts_calculation) +static void +test_redundant_pkt_does_not_affect_pts_calculation (GstRTPBufferFlags flags, + const JitterBufferSkewCtx * ctx) { GstHarness *h = gst_harness_new ("rtpjitterbuffer"); GstBuffer *buffer; guint next_seqnum; - guint rtx_seqnum; + guint redundant_pkt_seqnum; GstClockTime now; - const RtxSkewCtx *ctx = &rtx_does_not_affect_pts_calculation_input[__i__]; /* set up a deterministic state and take the time on the clock */ - g_object_set (h->element, "do-retransmission", TRUE, "do-lost", TRUE, NULL); + if (flags & GST_RTP_BUFFER_FLAG_RETRANSMISSION) + g_object_set (h->element, "do-retransmission", TRUE, "do-lost", TRUE, NULL); + next_seqnum = construct_deterministic_initial_state (h, 3000); now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); - /* push in a "bad" RTX buffer, arriving at various times / seqnums */ - rtx_seqnum = next_seqnum + ctx->seqnum_skew; - buffer = generate_test_buffer_full (now + ctx->dts_skew, rtx_seqnum, - rtx_seqnum * TEST_RTP_TS_DURATION); - GST_BUFFER_FLAG_SET (buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION); + /* push in a "bad" redundant buffer, arriving at various times / seqnums */ + redundant_pkt_seqnum = next_seqnum + ctx->seqnum_skew; + buffer = generate_test_buffer_full (now + ctx->dts_skew, redundant_pkt_seqnum, + redundant_pkt_seqnum * TEST_RTP_TS_DURATION); + GST_BUFFER_FLAG_SET (buffer, flags); gst_harness_push (h, buffer); /* now push in the next regular buffer at its ideal time, and verify the - rogue RTX-buffer did not mess things up */ + rogue RTX or ULPFEC buffer did not mess things up */ push_test_buffer (h, next_seqnum); now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); buffer = gst_harness_pull (h); @@ -3158,6 +3162,22 @@ GST_START_TEST (test_rtx_does_not_affect_pts_calculation) gst_harness_teardown (h); } +static const JitterBufferSkewCtx rtx_does_not_affect_pts_calculation_input[] = { + {0, 0}, + {20 * GST_MSECOND, -100}, + {20 * GST_MSECOND, 100}, + {-10 * GST_MSECOND, 1}, + {100 * GST_MSECOND, 0}, +}; + +GST_START_TEST (test_rtx_does_not_affect_pts_calculation) +{ + const JitterBufferSkewCtx *ctx = + &rtx_does_not_affect_pts_calculation_input[__i__]; + test_redundant_pkt_does_not_affect_pts_calculation + (GST_RTP_BUFFER_FLAG_RETRANSMISSION, ctx); +} + GST_END_TEST; GST_START_TEST (test_dont_drop_packet_based_on_skew) @@ -3699,6 +3719,182 @@ GST_START_TEST (test_early_rtcp_sr_allows_meta) GST_END_TEST; +GST_START_TEST (test_ulpfec_large_pkt_spacing) +{ + gint latency_ms = 20; + gint frame_dur_ms = 50; + gint i, seq; + GstBuffer *buffer; + GstClockTime now; + GstClockTime frame_dur = frame_dur_ms * GST_MSECOND; + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + + g_object_set (h->element, "do-lost", TRUE, "latency", latency_ms, NULL); + gst_harness_set_src_caps (h, generate_caps ()); + + /* Pushing 2 frames @frame_dur_ms ms apart from each other to initialize + * packet_spacing and avg jitter */ + for (seq = 0, now = 0; seq < 2; ++seq, now += frame_dur) { + gst_harness_set_time (h, now); + gst_harness_push (h, generate_test_buffer_full (now, seq, + AS_TEST_BUF_RTP_TIME (now))); + if (seq == 0) + gst_harness_crank_single_clock_wait (h); + buffer = gst_harness_pull (h); + fail_unless_equals_int64 (now, GST_BUFFER_PTS (buffer)); + gst_buffer_unref (buffer); + } + + /* drop GstEventStreamStart & GstEventCaps & GstEventSegment & Latency */ + for (i = 0; i < 4; i++) + gst_event_unref (gst_harness_pull_event (h)); + /* drop reconfigure event */ + gst_event_unref (gst_harness_pull_upstream_event (h)); + + /* Pushing packet #2 as ULPFEC */ + now = seq * frame_dur; + gst_harness_set_time (h, now); + buffer = generate_test_buffer_full (now, seq, AS_TEST_BUF_RTP_TIME (now)); + GST_BUFFER_FLAG_SET (buffer, GST_RTP_BUFFER_FLAG_ULPFEC); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buffer)); + buffer = gst_harness_pull (h); + fail_unless_equals_int64 (now, GST_BUFFER_PTS (buffer)); + gst_buffer_unref (buffer); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + + /* Packet #3 should have PTS not affected by clock skew logic */ + seq += 1; + now = seq * frame_dur; + gst_harness_set_time (h, now); + gst_harness_push (h, generate_test_buffer_full (now, seq, + AS_TEST_BUF_RTP_TIME (now))); + buffer = gst_harness_pull (h); + fail_unless_equals_int64 (now, GST_BUFFER_PTS (buffer)); + gst_buffer_unref (buffer); + + gst_harness_teardown (h); +} + +GST_END_TEST; + + +static const JitterBufferSkewCtx + ulpfec_pkt_does_not_affect_pts_calculation_input[] = { + {0, 0}, + {20 * GST_MSECOND, -100}, + {-10 * GST_MSECOND, 1}, + {100 * GST_MSECOND, 0}, +}; + +GST_START_TEST (test_ulpfec_does_not_affect_pts_calculation) +{ + const JitterBufferSkewCtx *ctx = + &ulpfec_pkt_does_not_affect_pts_calculation_input[__i__]; + + test_redundant_pkt_does_not_affect_pts_calculation + (GST_RTP_BUFFER_FLAG_ULPFEC, ctx); +} + +GST_END_TEST; + +typedef struct +{ + guint start_seqnum; + gint32 rtptime_diff_in_seqnum; +} JitterBufferSkewRtxCtx; + +static const JitterBufferSkewRtxCtx + ulpfec_jump_of_rtp_ts_does_not_drop_pkt_input[] = { + {0, 0}, + {100, 0}, + {100, 100}, + {100, 200}, + {100, -151}, + {100, -152}, + {100, -200} +}; + +GST_START_TEST (test_ulpfec_jump_of_rtp_ts_does_not_drop_pkt) +{ + const JitterBufferSkewRtxCtx *ctx = + &ulpfec_jump_of_rtp_ts_does_not_drop_pkt_input[__i__]; + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + GstBuffer *buffer = NULL; + guint next_seqnum = 0; + GstClockTime now = 0; + + /* Init jitterbuffer from N-th packet (value of 'next_seqnum'). That would make base_time of jitterbuffer + * starting from (N * TEST_BUF_MS) */ + next_seqnum = + construct_deterministic_initial_state_full (h, ctx->start_seqnum, 3000, + generate_caps ()); + now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); + + /* push in a "bad" ULPFEC buffer with skewed RTP timestamp */ + buffer = generate_test_buffer_full (now, next_seqnum, + ((next_seqnum + ctx->rtptime_diff_in_seqnum) * TEST_RTP_TS_DURATION)); + + GST_BUFFER_FLAG_SET (buffer, GST_RTP_BUFFER_FLAG_ULPFEC); + + gst_harness_push (h, buffer); + + /* expect ULPFEC buffer to pass through the rtpjitterbuffer */ + buffer = gst_harness_pull (h); + gst_buffer_unref (buffer); + next_seqnum++; + + /* now push in the next regular buffer at its ideal time, and verify the + rogue ULPFEC buffer did not mess things up */ + push_test_buffer (h, next_seqnum); + now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); + buffer = gst_harness_pull (h); + fail_unless_equals_int (next_seqnum, get_rtp_seq_num (buffer)); + fail_unless_equals_int64 (now, GST_BUFFER_PTS (buffer)); + + gst_buffer_unref (buffer); + + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_find_random_stall) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + guint i; + guint num_bufs = 10; + BufferArrayCtx *ctx = g_new0 (BufferArrayCtx, num_bufs); + for (i = 0; i < num_bufs; i++) { + ctx[i].seqnum_d = g_random_int_range (0, 300) - 20; + ctx[i].rtptime_d = g_random_int_range (0, 300) - 20; + ctx[i].rtptime_d *= TEST_RTP_TS_DURATION; + ctx[i].rtx = g_random_boolean (); + ctx[i].sleep_us = g_random_int_range (0, G_USEC_PER_SEC / 5); + } + + g_object_set (h->element, "do-lost", TRUE, + "do-retransmission", TRUE, + "rtx-next-seqnum", FALSE, + "rtx-delay-reorder", 0, + "rtx-max-retries", 5, + "rtx-min-retry-timeout", 50, "rtx-retry-period", 250, NULL); + + if (!check_for_stall (h, ctx, num_bufs)) { + for (i = 0; i < num_bufs; i++) { + g_print (" { %d, %d, %s, %d },\n", + ctx[i].seqnum_d, ctx[i].rtptime_d, + ctx[i].rtx ? "TRUE" : "FALSE", ctx[i].sleep_us); + } + fail_if (TRUE); + } + g_free (ctx); + + gst_harness_teardown (h); +} + +GST_END_TEST; + static Suite * rtpjitterbuffer_suite (void) { @@ -3784,6 +3980,13 @@ rtpjitterbuffer_suite (void) tcase_add_test (tc_chain, test_early_rtcp_sr_allows_meta); + tcase_add_test (tc_chain, test_ulpfec_large_pkt_spacing); + tcase_add_loop_test (tc_chain, test_ulpfec_jump_of_rtp_ts_does_not_drop_pkt, + 0, G_N_ELEMENTS (ulpfec_jump_of_rtp_ts_does_not_drop_pkt_input)); + tcase_add_loop_test (tc_chain, test_ulpfec_does_not_affect_pts_calculation, 0, + G_N_ELEMENTS (ulpfec_pkt_does_not_affect_pts_calculation_input)); + tcase_add_test (tc_chain, test_find_random_stall); + return s; } From cc3a9e768685da9952adc251a4bf2cdc9f416d92 Mon Sep 17 00:00:00 2001 From: "sergei.kovalev" Date: Thu, 16 Jun 2022 14:08:46 +0200 Subject: [PATCH 021/339] gstrtpjitterbuffer: Improve avg_jitter calculation. Ignore sequential packets with the same rtp time stamp during jitter estimation. --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 4f75df2c84b..f01216ac3bb 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -2831,9 +2831,9 @@ static void calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, guint32 rtptime) { - gint32 rtpdiff; - GstClockTimeDiff dtsdiff, rtpdiffns, diff; - gint factor; + gint32 rtpdiff = 0; + GstClockTimeDiff rtpdiffns = 0, diff = 0, dtsdiff = 0; + gint factor = 1; GstRtpJitterBufferPrivate *priv; priv = jitterbuffer->priv; @@ -2857,6 +2857,10 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, priv->equidistant += 1; priv->equidistant = CLAMP (priv->equidistant, -7, 7); + /* Non-equidistant packets are not used in jitter calculation */ + if (rtpdiff == 0) + return; + if (rtpdiff > 0) rtpdiffns = gst_util_uint64_scale_int (rtpdiff, GST_SECOND, priv->clock_rate); From b1ca2145b0f9e55ee055c680ff6eed8667544400 Mon Sep 17 00:00:00 2001 From: "sergei.kovalev" Date: Fri, 17 Jun 2022 22:00:58 +0200 Subject: [PATCH 022/339] gstrtpjitterbuffer: Add to JB stats latency_estimation --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 131 ++++++++++++- .../tests/check/elements/rtpjitterbuffer.c | 184 ++++++++++++++++++ 2 files changed, 314 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index f01216ac3bb..3bef88fc2ec 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -105,6 +105,7 @@ #include #include #include +#include #include #include #include @@ -450,6 +451,15 @@ struct _GstRtpJitterBufferPrivate GstClockTime total_avg_jitter; GstClockTime avg_jitter; + /* for the latency estimation */ + guint estimated_latency_ms; + GstClockTimeDiff avg_clock_diff; + guint64 instant_max_jitter_top; + guint64 instant_max_jitter_bottom; + guint64 etimatied_max_jitter_top; + guint64 etimatied_max_jitter_bottom; + gfloat dencity_ratio; + /* for dropped packet messages */ GstClockTime last_drop_msg_timestamp; /* accumulators; reset every time a drop message is posted */ @@ -620,6 +630,7 @@ static GQuark quark_total_avg_jitter; static GQuark quark_min_jitter; static GQuark quark_max_jitter; static GQuark quark_latency_ms; +static GQuark quark_estimated_latency_ms; static GQuark quark_rtx_count; static GQuark quark_rtx_success_count; static GQuark quark_rtx_per_packet; @@ -644,6 +655,8 @@ gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass) quark_total_avg_jitter = g_quark_from_static_string ("total-avg-jitter"); quark_min_jitter = g_quark_from_static_string ("min-jitter"); quark_max_jitter = g_quark_from_static_string ("max-jitter"); + quark_estimated_latency_ms = + g_quark_from_static_string ("estimated-latency-ms"); quark_latency_ms = g_quark_from_static_string ("latency-ms"); quark_rtx_count = g_quark_from_static_string ("rtx-count"); quark_rtx_success_count = g_quark_from_static_string ("rtx-success-count"); @@ -1188,6 +1201,13 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer) priv->last_known_ext_rtptime = -1; priv->last_known_ntpnstime = -1; priv->avg_jitter = 0; + priv->avg_clock_diff = -1; + priv->instant_max_jitter_top = 0; + priv->instant_max_jitter_bottom = 0; + priv->etimatied_max_jitter_top = 0; + priv->etimatied_max_jitter_bottom = 0; + priv->dencity_ratio = 0.5f; + priv->estimated_latency_ms = 0; priv->last_drop_msg_timestamp = GST_CLOCK_TIME_NONE; priv->num_too_late = 0; priv->num_drop_on_latency = 0; @@ -1784,6 +1804,13 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer) priv->avg_jitter = 0; priv->min_jitter = G_MAXINT64; priv->last_dts = -1; + priv->avg_clock_diff = -1; + priv->instant_max_jitter_top = 0; + priv->instant_max_jitter_bottom = 0; + priv->etimatied_max_jitter_top = 0; + priv->etimatied_max_jitter_bottom = 0; + priv->dencity_ratio = 0.5f; + priv->estimated_latency_ms = 0; priv->last_rtptime = -1; priv->last_ntpnstime = -1; priv->last_known_ext_rtptime = -1; @@ -2827,6 +2854,105 @@ gst_rtp_jitter_buffer_handle_missing_packets (GstRtpJitterBuffer * jitterbuffer, } } +/* Usually, the distribution of 'jitter' is non-symmetric across the mean value + * (it is similar to the gamma distribution) so the latency estimation based + * on six jitter standard deviations doesn't always give a good prediction. + * The purpose is to give a more precise latency estimation taking into + * account the asymmetric nature of the jitter. To do so we estimate two + * parameters - "etimatied_max_jitter_top", and "etimatied_max_jitter_bottom", + * each of them estimates maximum jitter above or below the mean. In sum, they + * give the total estimated latency. + * The idea is to outline the maximum observed jitters and interpolate them to + * the future. To do so we have two estimations "instant_max_jitter" and "etimatied_max_jitter". + * Both of them are increasing to the observed jitter value if it is bigger than + * their current value. Otherwise, they are decreasing. The "instant_max_jitter" + * is getting decreased to the observed jitter value with some linear factor. + * The "etimatied_max_jitter" is decreasing towards the "instant_max_jitter" + * with some non-linear factor. Non-linearity is added to smooth out the temporal + * fluctuations in "instant_max_jitter". + * */ +static inline void +max_jitter_estimation (guint64 jitter, guint64 * short_estimation, + guint64 * long_estimation, gfloat decrease_factor) +{ + const gint fast_decrease_factor = 64; + const gint slow_decrease_factor = 256; + gint64 diff = 0; + if (jitter >= *short_estimation) + *short_estimation = jitter; + else { + *short_estimation -= ((*short_estimation - jitter) / fast_decrease_factor); + } + diff = *long_estimation - *short_estimation; + if (diff <= 0) + *long_estimation = *short_estimation; + else { + guint delta = diff / slow_decrease_factor; + gfloat ratio = + 1.f - ((gfloat) (*short_estimation) / (gfloat) (*long_estimation)); + *long_estimation -= delta * pow (ratio, (16.f * decrease_factor)); + } +} + +static void +estimate_latency (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, + guint32 rtptime) +{ + gint64 jitter = 0; + GstClockTimeDiff clock_diff = 0; + GstRtpJitterBufferPrivate *priv = jitterbuffer->priv; + + if (G_UNLIKELY (dts == GST_CLOCK_TIME_NONE) || priv->clock_rate <= 0) + return; + + /* clock_diff - consists Cd + D + J where: + * Cd - initial clock difference between sender and recover + * D - constant delay. + * J - jitter */ + clock_diff = + ((gint64) dts - (gint64) gst_util_uint64_scale_int (rtptime, GST_SECOND, + priv->clock_rate)); + if (priv->avg_clock_diff == -1) { + priv->avg_clock_diff = clock_diff; + return; + } + + /* By averaging clock_diff we getting (Cd + D) + Jm + * where 'Jm' is mean of the jitter */ + priv->avg_clock_diff -= + ((priv->avg_clock_diff - clock_diff) / (gint) MIN (128, MAX (1, + priv->jitter_count))); + + /* Jitter is a deviation form the average clock_diff. */ + jitter = clock_diff - priv->avg_clock_diff; + + /* Purpouse of 'dencity_ratio' is to represent assimmetric distrebution of the jitter. + * The bigger 'dencity_ratio' the faster decreasing the bottom jitter estimation, + * but top estimation slower. If it 0.5 both top and bottom has equal decreasing factor. */ + priv->dencity_ratio -= + ((priv->dencity_ratio - (gfloat) (clock_diff < + priv->avg_clock_diff)) / 16.f); + priv->dencity_ratio = MIN (1., MAX (0., priv->dencity_ratio)); + + /* Update estimations of max jiiter above the jitter mean. */ + max_jitter_estimation (MAX (0, jitter), &priv->instant_max_jitter_top, + &priv->etimatied_max_jitter_top, priv->dencity_ratio); + + /* Update estimations of max jiiter below the jitter mean. */ + max_jitter_estimation (ABS (MIN (0, jitter)), + &priv->instant_max_jitter_bottom, &priv->etimatied_max_jitter_bottom, + (1.f - priv->dencity_ratio)); + + priv->estimated_latency_ms = + (priv->etimatied_max_jitter_top + + priv->etimatied_max_jitter_bottom) / GST_MSECOND; + + GST_LOG_OBJECT (jitterbuffer, + "estimated_latency:%" GST_TIME_FORMAT, + GST_TIME_ARGS (priv->etimatied_max_jitter_top + + priv->etimatied_max_jitter_bottom)); +} + static void calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, guint32 rtptime) @@ -3276,8 +3402,10 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, if (G_UNLIKELY (priv->eos)) goto have_eos; - if (!is_rtx && !is_ulpfec) + if (!is_rtx && !is_ulpfec) { calculate_jitter (jitterbuffer, dts, rtptime); + estimate_latency (jitterbuffer, dts, rtptime); + } if (priv->seqnum_base != -1) { gint gap; @@ -5447,6 +5575,7 @@ gst_rtp_jitter_buffer_create_stats (GstRtpJitterBuffer * jbuf) quark_total_avg_jitter, G_TYPE_UINT64, priv->total_avg_jitter, quark_min_jitter, G_TYPE_UINT64, priv->min_jitter, quark_max_jitter, G_TYPE_UINT64, priv->max_jitter, + quark_estimated_latency_ms, G_TYPE_UINT, priv->estimated_latency_ms, quark_latency_ms, G_TYPE_UINT, priv->latency_ms, quark_rtx_count, G_TYPE_UINT64, priv->num_rtx_requests, quark_rtx_success_count, G_TYPE_UINT64, priv->num_rtx_success, diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c index 707e30e41cd..1cd9200d7bb 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c @@ -3895,6 +3895,185 @@ GST_START_TEST (test_find_random_stall) GST_END_TEST; +static void +validate_estimated_latency (GstHarness * h, guint expected_estimated_latency_ms) +{ + GstStructure *s = NULL; + gint diff = 0; + guint actual_estimated_latency_ms = 0; + g_object_get (h->element, "stats", &s, NULL); + fail_unless (gst_structure_get_uint (s, "estimated-latency-ms", + &actual_estimated_latency_ms)); + + diff = (actual_estimated_latency_ms - expected_estimated_latency_ms); + fail_unless ((diff == 0), + "Actual 'estimated-latency-ms' (%" GST_TIME_FORMAT + ") is off from expected value (%" GST_TIME_FORMAT ") " "by %" + GST_STIME_FORMAT ".", + GST_TIME_ARGS (actual_estimated_latency_ms * GST_MSECOND), + GST_TIME_ARGS (expected_estimated_latency_ms * GST_MSECOND), + GST_STIME_ARGS (diff * GST_MSECOND)); + gst_structure_free (s); +} + + +static gint +buffer_array_append_sequential_with_jitter (GArray * array, + gint * jitter_array_ms, guint num_bufs, gint prev_jitter) +{ + guint i; + for (i = 0; i < num_bufs; i++) { + BufferArrayCtx ctx; + ctx.seqnum_d = 1; + ctx.rtptime_d = TEST_RTP_TS_DURATION; /* 20ms for 8KHz */ + ctx.rtx = FALSE; + ctx.sleep_us = + (G_USEC_PER_SEC / 1000 * (TEST_BUF_MS - prev_jitter + + jitter_array_ms[i])); + g_array_append_val (array, ctx); + prev_jitter = jitter_array_ms[i]; + } + return prev_jitter; +} + +static void +buffer_array_push_fast (GstHarness * h, GArray * array, + guint16 seqnum_base, guint32 rtptime_base) +{ + guint16 seqnum = seqnum_base; + guint32 rtptime = rtptime_base; + GstClockTime now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); + guint i; + + for (i = 0; i < array->len; i++) { + BufferArrayCtx *ctx = &g_array_index (array, BufferArrayCtx, i); + GstBuffer *buf = generate_test_buffer_full (now, seqnum, rtptime); + if (ctx->rtx) + GST_BUFFER_FLAG_SET (buf, GST_RTP_BUFFER_FLAG_RETRANSMISSION); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + seqnum += ctx->seqnum_d; + rtptime += ctx->rtptime_d; + now += (GST_USECOND * ctx->sleep_us); + } +} + +static void +push_buf_array (GstHarness * h, GArray * array) +{ + guint16 base_seqnum = 10000; + guint32 base_rtptime = base_seqnum * TEST_RTP_TS_DURATION; + gst_harness_set_src_caps (h, generate_caps ()); + buffer_array_push_fast (h, array, base_seqnum, base_rtptime); +} + +GST_START_TEST (test_latency_estimation_no_jitter) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + GArray *array = g_array_new (FALSE, FALSE, sizeof (BufferArrayCtx)); + + g_object_set (h->element, "latency", 0, NULL); + buffer_array_append_sequential (array, 128); + push_buf_array (h, array); + validate_estimated_latency (h, 0); + + g_array_set_size (array, 0); + g_array_unref (array); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_latency_estimation_with_jitter) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + gint jitter_array_ms[] = { 1, 4, -3, 5, 3, -2, 7 }; + GArray *array = g_array_new (FALSE, FALSE, sizeof (BufferArrayCtx)); + + g_object_set (h->element, "latency", 0, NULL); + + buffer_array_append_sequential (array, 128); + /* Push packets with jitter and validate latency estimation */ + buffer_array_append_sequential_with_jitter (array, jitter_array_ms, + G_N_ELEMENTS (jitter_array_ms), 0); + push_buf_array (h, array); + validate_estimated_latency (h, 7); + g_array_set_size (array, 0); + + /* Push more packets without jitter and validate + * latency estimation decreasing */ + buffer_array_append_sequential (array, 256); + push_buf_array (h, array); + validate_estimated_latency (h, 5); + + g_array_set_size (array, 0); + g_array_unref (array); + gst_harness_teardown (h); +} + +GST_END_TEST; + + +GST_START_TEST (test_latency_estimation_increasing_delay) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + gint jitter_array_ms[] = { 1 }; + GArray *array = g_array_new (FALSE, FALSE, sizeof (BufferArrayCtx)); + + g_object_set (h->element, "latency", 0, NULL); + + buffer_array_append_sequential (array, 128); + + for (size_t i = 0; i < 2000; i++) { + buffer_array_append_sequential_with_jitter (array, jitter_array_ms, + G_N_ELEMENTS (jitter_array_ms), 0); + } + + /* All buffers with ever-increasing delay */ + push_buf_array (h, array); + /* This is unusual usecase but latency estimation + * should not blow up */ + validate_estimated_latency (h, 127); + + g_array_set_size (array, 0); + g_array_unref (array); + gst_harness_teardown (h); +} + +GST_END_TEST; + + +GST_START_TEST (test_latency_estimation_reaction_on_outlyer) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + gint jitter_array_ms[] = { 1, -1, 100, -1, }; + GArray *array = g_array_new (FALSE, FALSE, sizeof (BufferArrayCtx)); + + g_object_set (h->element, "latency", 0, NULL); + + buffer_array_append_sequential (array, 128); + + buffer_array_append_sequential_with_jitter (array, jitter_array_ms, + G_N_ELEMENTS (jitter_array_ms), 0); + /* Push buffers with one outlayer (100ms late) */ + push_buf_array (h, array); + /* Validat latency estimation */ + validate_estimated_latency (h, 100); + + g_array_set_size (array, 0); + /* Push more buffers and check that we reduce + * latency estimation back to 0 */ + buffer_array_append_sequential (array, 1500); + push_buf_array (h, array); + validate_estimated_latency (h, 0); + + g_array_set_size (array, 0); + g_array_unref (array); + gst_harness_teardown (h); +} + +GST_END_TEST; + + static Suite * rtpjitterbuffer_suite (void) { @@ -3985,7 +4164,12 @@ rtpjitterbuffer_suite (void) 0, G_N_ELEMENTS (ulpfec_jump_of_rtp_ts_does_not_drop_pkt_input)); tcase_add_loop_test (tc_chain, test_ulpfec_does_not_affect_pts_calculation, 0, G_N_ELEMENTS (ulpfec_pkt_does_not_affect_pts_calculation_input)); + tcase_add_test (tc_chain, test_find_random_stall); + tcase_add_test (tc_chain, test_latency_estimation_no_jitter); + tcase_add_test (tc_chain, test_latency_estimation_with_jitter); + tcase_add_test (tc_chain, test_latency_estimation_increasing_delay); + tcase_add_test (tc_chain, test_latency_estimation_reaction_on_outlyer); return s; } From 7aa3fd803d9e5065e1e56184d019138b8c54a623 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Thu, 17 Nov 2022 21:37:45 +0900 Subject: [PATCH 023/339] good/jitterbuffer: initialize SSRC on RTCP chain method It is possible to log out an uninitialized SSRC. Reported as by clang's scan build. --- .../gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 3bef88fc2ec..0ad408c6e4b 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -4901,7 +4901,7 @@ gst_rtp_jitter_buffer_chain_rtcp (GstPad * pad, GstObject * parent, GstRtpJitterBuffer *jitterbuffer; GstRtpJitterBufferPrivate *priv; GstFlowReturn ret = GST_FLOW_OK; - guint32 ssrc; + guint32 ssrc = 0x0; GstRTCPPacket packet; guint64 ext_rtptime, ntptime; GstClockTime ntpnstime = GST_CLOCK_TIME_NONE; From 5e69df5c4b10e5cde4c9bee96e168866288e9050 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 16 Mar 2023 18:32:50 +0100 Subject: [PATCH 024/339] rtpjitterbuffer: new packet spacing heuristic - use RTP time instead of pts, the former is based on the sender clock and so provides a more precise metric. - instead of averaging spacings, update the packet spacing when we receive two consecutive identical spacings, discarding decreasing RTP times. This new heuristic should provide a more precise and constant packet spacing on audio streams. It will allow us to support discontinuous transmission (DTX) as the large spacings introduced by DTX will no longer increase the actual packet spacing. Co-Authored-By: Tulio Beloqui (cherry picked from commit f68e9b9897a1e52233dea35612c398375fa4f8b7) --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 72 ++++++++--------- .../tests/check/elements/rtpjitterbuffer.c | 81 ++++++++++++++++++- 2 files changed, 113 insertions(+), 40 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 0ad408c6e4b..3964fc532b8 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -371,10 +371,14 @@ struct _GstRtpJitterBufferPrivate guint32 seqnum_base; /* last output time */ GstClockTime last_out_time; - /* last valid input timestamp and rtptime pair */ - GstClockTime ips_pts; guint64 ips_rtptime; + /* current packet spacing */ GstClockTime packet_spacing; + /* packet spacing calculated from the last received packet. + * Will update packet_spacing if we receive 2 consecutives packets with the + * same spacing. */ + GstClockTime prev_packet_spacing; + gint equidistant; GQueue gap_packets; @@ -1739,6 +1743,11 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, gst_rtp_get_extmap_id_for_attribute (caps_struct, GST_RTP_HDREXT_BASE GST_RTP_HDREXT_NTP_64); + /* reset packet spacing */ + priv->packet_spacing = 0; + priv->prev_packet_spacing = GST_CLOCK_TIME_NONE; + priv->ips_rtptime = -1; + return TRUE; /* ERRORS */ @@ -1790,8 +1799,8 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer) priv->next_seqnum = -1; priv->seqnum_base = -1; priv->ips_rtptime = -1; - priv->ips_pts = GST_CLOCK_TIME_NONE; priv->packet_spacing = 0; + priv->prev_packet_spacing = GST_CLOCK_TIME_NONE; priv->next_in_seqnum = -1; priv->clock_rate = -1; priv->ntp64_ext_id = 0; @@ -2585,42 +2594,38 @@ update_rtx_timers (GstRtpJitterBuffer * jitterbuffer, guint16 seqnum, } static void -calculate_packet_spacing (GstRtpJitterBuffer * jitterbuffer, guint32 rtptime, - GstClockTime pts) +calculate_packet_spacing (GstRtpJitterBuffer * jitterbuffer, guint32 rtptime) { GstRtpJitterBufferPrivate *priv = jitterbuffer->priv; /* we need consecutive seqnums with a different * rtptime to estimate the packet spacing. */ if (priv->ips_rtptime != rtptime) { - /* rtptime changed, check pts diff */ - if (priv->ips_pts != -1 && pts != -1 && pts > priv->ips_pts) { - GstClockTime new_packet_spacing = pts - priv->ips_pts; - GstClockTime old_packet_spacing = priv->packet_spacing; - - /* Biased towards bigger packet spacings to prevent - * too many unneeded retransmission requests for next - * packets that just arrive a little later than we would - * expect */ - if (old_packet_spacing > new_packet_spacing) - priv->packet_spacing = - (new_packet_spacing + 3 * old_packet_spacing) / 4; - else if (old_packet_spacing > 0) - priv->packet_spacing = - (3 * new_packet_spacing + old_packet_spacing) / 4; - else - priv->packet_spacing = new_packet_spacing; + /* rtptime changed */ + if (priv->clock_rate > 0 && rtptime > priv->ips_rtptime) { + guint64 rtpdiff; + GstClockTime new_packet_spacing; + + rtpdiff = rtptime - (guint32) priv->ips_rtptime; + new_packet_spacing = + gst_util_uint64_scale_int (rtpdiff, GST_SECOND, priv->clock_rate); + + if (new_packet_spacing != priv->packet_spacing) { + if (priv->prev_packet_spacing == new_packet_spacing + || !GST_CLOCK_TIME_IS_VALID (priv->prev_packet_spacing)) { + /* we received two consecutive packets with the same spacing, update */ + GST_DEBUG_OBJECT (jitterbuffer, + "new packet spacing %" GST_TIME_FORMAT + " old packet spacing %" GST_TIME_FORMAT, + GST_TIME_ARGS (new_packet_spacing), + GST_TIME_ARGS (priv->packet_spacing)); - GST_DEBUG_OBJECT (jitterbuffer, - "new packet spacing %" GST_TIME_FORMAT - " old packet spacing %" GST_TIME_FORMAT - " combined to %" GST_TIME_FORMAT, - GST_TIME_ARGS (new_packet_spacing), - GST_TIME_ARGS (old_packet_spacing), - GST_TIME_ARGS (priv->packet_spacing)); + priv->packet_spacing = new_packet_spacing; + } + priv->prev_packet_spacing = new_packet_spacing; + } } priv->ips_rtptime = rtptime; - priv->ips_pts = pts; } } @@ -3187,13 +3192,10 @@ gst_rtp_jitter_buffer_reset (GstRtpJitterBuffer * jitterbuffer, /* reset spacing estimation when gap */ priv->ips_rtptime = -1; - priv->ips_pts = GST_CLOCK_TIME_NONE; buffers = g_list_copy (priv->gap_packets.head); g_queue_clear (&priv->gap_packets); - priv->ips_rtptime = -1; - priv->ips_pts = GST_CLOCK_TIME_NONE; JBUF_UNLOCK (jitterbuffer->priv); for (l = buffers; l; l = l->next) { @@ -3481,7 +3483,6 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, do_next_seqnum = TRUE; /* take rtptime and pts to calculate packet spacing */ priv->ips_rtptime = rtptime; - priv->ips_pts = pts; } else { gint gap; @@ -3527,7 +3528,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, if (G_LIKELY (gap == 0)) { /* packet is expected */ - calculate_packet_spacing (jitterbuffer, rtptime, pts); + calculate_packet_spacing (jitterbuffer, rtptime); do_next_seqnum = TRUE; } else { @@ -3555,7 +3556,6 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, /* reset spacing estimation when gap */ priv->ips_rtptime = -1; - priv->ips_pts = GST_CLOCK_TIME_NONE; } } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c index 1cd9200d7bb..5f1a6ae2e44 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c @@ -666,7 +666,7 @@ construct_deterministic_initial_state_full (GstHarness * h, g_assert (latency_ms % TEST_BUF_MS == 0); - gst_harness_set_src_caps (h, generate_caps ()); + gst_harness_set_src_caps (h, caps); g_object_set (h->element, "latency", latency_ms, NULL); /* When the first packet arrives in the jitterbuffer, it will create a @@ -2407,10 +2407,9 @@ GST_START_TEST (test_rtx_with_backwards_rtptime) * we need to advance the clock to the expected point */ gst_harness_wait_for_clock_id_waits (h, 1, 1); - gst_harness_set_time (h, 6 * TEST_BUF_DURATION + 15 * GST_MSECOND); + gst_harness_set_time (h, 6 * TEST_BUF_DURATION); gst_harness_crank_single_clock_wait (h); - verify_rtx_event (h, 6, 5 * TEST_BUF_DURATION + 15 * GST_MSECOND, - 20, 35 * GST_MSECOND); + verify_rtx_event (h, 6, 5 * TEST_BUF_DURATION, 20, 20 * GST_MSECOND); fail_unless (verify_jb_stats (h->element, gst_structure_new ("application/x-rtp-jitterbuffer-stats", @@ -2940,6 +2939,78 @@ GST_START_TEST (test_big_gap_arrival_time) GST_END_TEST; +static void +test_large_packet_spacing_event_pkt_pts (gboolean do_rtx) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + gint i; + guint seqnum_org; + GstClockTime dts_base; + guint seqnum_base, expected_seqnum; + guint32 rtpts_base; + GstClockTime expected_ts; + const gint num_consecutive = 3; + + g_object_set (h->element, "do-lost", TRUE, "do-retransmission", do_rtx, NULL); + seqnum_org = construct_deterministic_initial_state (h, 100); + + /* packets start coming after 10 seconds */ + dts_base = seqnum_org * 10 * GST_SECOND; + seqnum_base = seqnum_org; + rtpts_base = seqnum_base * TEST_RTP_TS_DURATION; + + for (i = 0; i < num_consecutive; i++) { + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, + generate_test_buffer_full (dts_base + i * TEST_BUF_DURATION, + seqnum_base + i, rtpts_base + i * TEST_RTP_TS_DURATION))); + } + + for (i = 0; i < num_consecutive; i++) { + GstBuffer *buf = gst_harness_pull (h); + guint expected_seqnum = seqnum_base + i; + fail_unless_equals_int (expected_seqnum, get_rtp_seq_num (buf)); + + expected_ts = dts_base + i * TEST_BUF_DURATION; + fail_unless_equals_int (expected_ts, GST_BUFFER_PTS (buf)); + gst_buffer_unref (buf); + } + + /* generate a gap of 1 */ + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, + generate_test_buffer_full (dts_base + + num_consecutive * TEST_BUF_DURATION, + seqnum_base + num_consecutive + 1, + rtpts_base + num_consecutive * TEST_RTP_TS_DURATION))); + + /* demand we generated a lost event or RTX request with the correct pts */ + expected_seqnum = seqnum_base + num_consecutive; + expected_ts = dts_base + (num_consecutive * TEST_BUF_DURATION); + gst_harness_crank_single_clock_wait (h); + if (do_rtx) { + gst_harness_crank_single_clock_wait (h); + verify_rtx_event (h, expected_seqnum, expected_ts, 100, TEST_BUF_DURATION); + } else { + verify_lost_event (h, expected_seqnum, expected_ts, 0); + } + + gst_harness_teardown (h); +} + +GST_START_TEST (test_large_packet_spacing_lost_pkt_pts) +{ + test_large_packet_spacing_event_pkt_pts (FALSE); +} + +GST_END_TEST; + + +GST_START_TEST (test_large_packet_spacing_rtx) +{ + test_large_packet_spacing_event_pkt_pts (TRUE); +} + +GST_END_TEST; + typedef struct { guint seqnum_offset; @@ -4137,6 +4208,8 @@ rtpjitterbuffer_suite (void) tcase_add_loop_test (tc_chain, test_big_gap_seqnum, 0, G_N_ELEMENTS (big_gap_testdata)); tcase_add_test (tc_chain, test_big_gap_arrival_time); + tcase_add_test (tc_chain, test_large_packet_spacing_lost_pkt_pts); + tcase_add_test (tc_chain, test_large_packet_spacing_rtx); tcase_add_test (tc_chain, test_fill_queue); tcase_add_loop_test (tc_chain, From e9f0efd51add112d5300690966929f7f8df104c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Thu, 27 Apr 2023 01:33:12 +0200 Subject: [PATCH 025/339] rtpjitterbuffer: test for non-utf8 cname Co-authored-by: John Bassett (cherry picked from commit 03e16b70f3ff61c2fc2f5bf4e84106e5bf38b27b) --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 10 ++++-- .../tests/check/elements/rtpjitterbuffer.c | 34 +++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 3964fc532b8..ef92ba3f7ec 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -4992,9 +4992,13 @@ gst_rtp_jitter_buffer_chain_rtcp (GstPad * pad, GstObject * parent, goto empty_buffer; JBUF_LOCK (priv); - if (cname) - insert_cname_ssrc_mapping (jitterbuffer, cname, ssrc); - + if (cname) { + if (g_utf8_validate (cname, -1, NULL)) { + insert_cname_ssrc_mapping (jitterbuffer, cname, ssrc); + } else { + GST_WARNING ("ignoring CNAME with non-utf8 data %s", cname); + } + } /* convert the RTP timestamp to our extended timestamp, using the same offset * we used in the jitterbuffer */ ext_rtptime = priv->jbuf->ext_rtptime; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c index 5f1a6ae2e44..a6122e45bbd 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c @@ -4144,6 +4144,38 @@ GST_START_TEST (test_latency_estimation_reaction_on_outlyer) GST_END_TEST; +GST_START_TEST (test_rtcp_non_utf8_cname) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + GstHarness *h_rtcp = gst_harness_new_with_element (h->element, "sink_rtcp", NULL); + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + GstRTCPPacket packet; + GstBuffer *rtcp_buf; + guint8 cname[] = { 0xf0, 0x28, 0x8c, 0x28, 0xf0, 0x28, 0x8c, 0x28, 0x00 }; + + gst_harness_set_src_caps (h, generate_caps ()); + gst_harness_set_src_caps_str (h_rtcp, "application/x-rtcp"); + + /* push in a SDES with an non-utf8 cname */ + rtcp_buf = gst_rtcp_buffer_new (1000); + fail_unless (gst_rtcp_buffer_map (rtcp_buf, GST_MAP_READWRITE, &rtcp)); + fail_unless (gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_SR, &packet)); + gst_rtcp_packet_sr_set_sender_info (&packet, TEST_BUF_SSRC, 0, 0, 0, 0); + fail_unless (gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_SDES, + &packet)); + fail_unless (gst_rtcp_packet_sdes_add_item (&packet, TEST_BUF_SSRC)); + fail_unless (gst_rtcp_packet_sdes_add_entry (&packet, GST_RTCP_SDES_CNAME, + sizeof (cname), cname)); + gst_rtcp_buffer_unmap (&rtcp); + gst_harness_push (h_rtcp, rtcp_buf); + + /* now push in a normal buffer, and don't explode */ + gst_harness_push (h, generate_test_buffer (0)); + + gst_harness_teardown (h_rtcp); + gst_harness_teardown (h); +} +GST_END_TEST; static Suite * rtpjitterbuffer_suite (void) @@ -4244,6 +4276,8 @@ rtpjitterbuffer_suite (void) tcase_add_test (tc_chain, test_latency_estimation_increasing_delay); tcase_add_test (tc_chain, test_latency_estimation_reaction_on_outlyer); + tcase_add_test (tc_chain, test_rtcp_non_utf8_cname); + return s; } From 8d93b5f08e5ee613982df0f8b8fead447e3707ad Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 7 Aug 2015 16:50:34 +0200 Subject: [PATCH 026/339] rtpbin: Don't sync streams with ntptime 0 A sender that has no notion of wallclock or elapsed time MAY set the NTP timestamp to zero. In this case the streams should not be synced because there is no fixed reference. --- .../gst-plugins-good/gst/rtpmanager/gstrtpbin.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c index d5adf0f64a3..2ca08d9428b 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c @@ -1511,9 +1511,9 @@ gst_rtp_bin_associate (GstRtpBin * bin, GstRtpBinStream * stream, guint8 len, running_time_rtp = last_extrtptime - base_rtptime; GST_DEBUG_OBJECT (bin, - "base %" G_GUINT64_FORMAT ", extrtptime %" G_GUINT64_FORMAT - ", local RTP %" G_GUINT64_FORMAT ", clock-rate %d, " - "clock-base %" G_GINT64_FORMAT, base_rtptime, + "base_time %" G_GUINT64_FORMAT ", base_rtptime %" G_GUINT64_FORMAT + ", extrtptime %" G_GUINT64_FORMAT ", local RTP %" G_GUINT64_FORMAT + ", clock-rate %d, clock-base %" G_GINT64_FORMAT, base_time, base_rtptime, last_extrtptime, running_time_rtp, clock_rate, rtp_clock_base); /* calculate local RTP time in gstreamer timestamp, we essentially perform the @@ -1524,7 +1524,10 @@ gst_rtp_bin_associate (GstRtpBin * bin, GstRtpBinStream * stream, guint8 len, gst_util_uint64_scale_int (running_time_rtp, GST_SECOND, clock_rate); running_time += base_time; - stream->have_sync = TRUE; + /* A sender that has no notion of wallclock or elapsed time MAY set the NTP + timestamp to zero. It makes sense to not synchronize this stream.*/ + if (ntpnstime != 0) + stream->have_sync = TRUE; GST_DEBUG_OBJECT (bin, "SR RTP running time %" G_GUINT64_FORMAT ", SR NTP %" G_GUINT64_FORMAT, From 6a047799d9a31941bc2f1d3894c9ae5dc65e2798 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 10 Mar 2017 12:10:33 +0100 Subject: [PATCH 027/339] rtpbin: Add action signal 'clear-pt-map-for-session-id' It's useful to be able to clear the pt map only for a single session. Although GstRtpSession alreay has 'clear-pt-map', this is not enough to clear the pt-map completey since it also must be done on GstRtpJitterbuffer and GstRtpPtDemux. --- .../gst/rtpmanager/gstrtpbin.c | 75 ++++++++++++++----- .../gst/rtpmanager/gstrtpbin.h | 1 + 2 files changed, 58 insertions(+), 18 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c index 2ca08d9428b..d0bd4ef6589 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c @@ -288,6 +288,7 @@ enum SIGNAL_REQUEST_PT_MAP, SIGNAL_PAYLOAD_TYPE_CHANGE, SIGNAL_CLEAR_PT_MAP, + SIGNAL_CLEAR_PT_MAP_FOR_SESSION_ID, SIGNAL_RESET_SYNC, SIGNAL_GET_SESSION, SIGNAL_GET_INTERNAL_SESSION, @@ -1059,32 +1060,40 @@ gst_rtp_bin_reset_sync (GstRtpBin * rtpbin) GST_RTP_BIN_UNLOCK (rtpbin); } +static void +gst_rtp_bin_clear_pt_map_for_session (GstRtpBin * bin, + GstRtpBinSession * session) +{ + GSList *streams; + + GST_DEBUG_OBJECT (bin, "clearing session %p", session); + g_signal_emit_by_name (session->session, "clear-pt-map", NULL); + + GST_RTP_SESSION_LOCK (session); + g_hash_table_foreach_remove (session->ptmap, return_true, NULL); + + for (streams = session->streams; streams; streams = g_slist_next (streams)) { + GstRtpBinStream *stream = (GstRtpBinStream *) streams->data; + + GST_DEBUG_OBJECT (bin, "clearing stream %p", stream); + if (g_signal_lookup ("clear-pt-map", G_OBJECT_TYPE (stream->buffer)) != 0) + g_signal_emit_by_name (stream->buffer, "clear-pt-map", NULL); + if (stream->demux) + g_signal_emit_by_name (stream->demux, "clear-pt-map", NULL); + } + GST_RTP_SESSION_UNLOCK (session); +} + static void gst_rtp_bin_clear_pt_map (GstRtpBin * bin) { - GSList *sessions, *streams; + GSList *sessions; GST_RTP_BIN_LOCK (bin); GST_DEBUG_OBJECT (bin, "clearing pt map"); for (sessions = bin->sessions; sessions; sessions = g_slist_next (sessions)) { GstRtpBinSession *session = (GstRtpBinSession *) sessions->data; - - GST_DEBUG_OBJECT (bin, "clearing session %p", session); - g_signal_emit_by_name (session->session, "clear-pt-map", NULL); - - GST_RTP_SESSION_LOCK (session); - g_hash_table_foreach_remove (session->ptmap, return_true, NULL); - - for (streams = session->streams; streams; streams = g_slist_next (streams)) { - GstRtpBinStream *stream = (GstRtpBinStream *) streams->data; - - GST_DEBUG_OBJECT (bin, "clearing stream %p", stream); - if (g_signal_lookup ("clear-pt-map", G_OBJECT_TYPE (stream->buffer)) != 0) - g_signal_emit_by_name (stream->buffer, "clear-pt-map", NULL); - if (stream->demux) - g_signal_emit_by_name (stream->demux, "clear-pt-map", NULL); - } - GST_RTP_SESSION_UNLOCK (session); + gst_rtp_bin_clear_pt_map_for_session (bin, session); } GST_RTP_BIN_UNLOCK (bin); @@ -1092,6 +1101,20 @@ gst_rtp_bin_clear_pt_map (GstRtpBin * bin) gst_rtp_bin_reset_sync (bin); } +static void +gst_rtp_bin_clear_pt_map_for_session_id (GstRtpBin * bin, guint session_id) +{ + GstRtpBinSession *session; + + GST_RTP_BIN_LOCK (bin); + GST_DEBUG_OBJECT (bin, "retrieving GstRtpSession, index: %u", session_id); + session = find_session_by_id (bin, (gint) session_id); + if (session) { + gst_rtp_bin_clear_pt_map_for_session (bin, session); + } + GST_RTP_BIN_UNLOCK (bin); +} + static GstElement * gst_rtp_bin_get_session (GstRtpBin * bin, guint session_id) { @@ -2234,6 +2257,20 @@ gst_rtp_bin_class_init (GstRtpBinClass * klass) G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (GstRtpBinClass, clear_pt_map), NULL, NULL, NULL, G_TYPE_NONE, 0, G_TYPE_NONE); + /** + * GstRtpBin::clear-pt-map-for-session-id: + * @rtpbin: the object which received the signal + * @id: the session id + * + * Clear all previously cached pt-mapping obtained with + * #GstRtpBin::request-pt-map for session @id. + */ + gst_rtp_bin_signals[SIGNAL_CLEAR_PT_MAP_FOR_SESSION_ID] = + g_signal_new ("clear-pt-map-for-session-id", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (GstRtpBinClass, + clear_pt_map_for_session_id), NULL, NULL, + g_cclosure_marshal_generic, G_TYPE_NONE, 1, G_TYPE_UINT); + /** * GstRtpBin::reset-sync: * @rtpbin: the object which received the signal @@ -3059,6 +3096,8 @@ gst_rtp_bin_class_init (GstRtpBinClass * klass) gstbin_class->handle_message = GST_DEBUG_FUNCPTR (gst_rtp_bin_handle_message); klass->clear_pt_map = GST_DEBUG_FUNCPTR (gst_rtp_bin_clear_pt_map); + klass->clear_pt_map_for_session_id = + GST_DEBUG_FUNCPTR (gst_rtp_bin_clear_pt_map_for_session_id); klass->reset_sync = GST_DEBUG_FUNCPTR (gst_rtp_bin_reset_sync); klass->get_session = GST_DEBUG_FUNCPTR (gst_rtp_bin_get_session); klass->get_internal_session = diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.h index bc7c7e5f66d..c2d0b653236 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.h @@ -120,6 +120,7 @@ struct _GstRtpBinClass { /* action signals */ void (*clear_pt_map) (GstRtpBin *rtpbin); + void (*clear_pt_map_for_session_id) (GstRtpBin *rtpbin, guint session); void (*reset_sync) (GstRtpBin *rtpbin); GstElement* (*get_session) (GstRtpBin *rtpbin, guint session); RTPSession* (*get_internal_session) (GstRtpBin *rtpbin, guint session); From 211d3bee675e04e1d08bf7b729ca1faeeb4e3208 Mon Sep 17 00:00:00 2001 From: Hani Mustafa Date: Fri, 18 Jul 2014 12:31:07 +0200 Subject: [PATCH 028/339] rtpmanager: Fix disabling of probation when receiving RTCP Probation = 0 (MIN_SEQUENTIAL) in RFC3550 is not really defined. Instead, the code expects that probation is 1 if you'd like to disable it. Concretely, the wraparound is not handled in the case when probation is set to 0 and the first packet has seq 0. --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c | 4 ++-- subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 6a02b40d0b5..cdc2b4ee048 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -2014,7 +2014,7 @@ obtain_source (RTPSession * sess, guint32 ssrc, gboolean * created, /* for RTP packets we need to set the source in probation. Receiving RTCP * packets of an SSRC, on the other hand, is a strong indication that we * are dealing with a valid source. */ - g_object_set (source, "probation", rtp ? sess->probation : 0, + g_object_set (source, "probation", rtp ? sess->probation : RTP_NO_PROBATION, "max-dropout-time", sess->max_dropout_time, "max-misorder-time", sess->max_misorder_time, NULL); @@ -2040,7 +2040,7 @@ obtain_source (RTPSession * sess, guint32 ssrc, gboolean * created, /* Receiving RTCP packets of an SSRC is a strong indication that we * are dealing with a valid source. */ if (!rtp) - g_object_set (source, "probation", 0, NULL); + g_object_set (source, "probation", RTP_NO_PROBATION, NULL); } /* update last activity */ source->last_activity = pinfo->current_time; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h index f099e4b7925..9b2cbd1ba35 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h @@ -31,7 +31,7 @@ /* the default number of consecutive RTP packets we need to receive before the * source is considered valid */ -#define RTP_NO_PROBATION 0 +#define RTP_NO_PROBATION 1 #define RTP_DEFAULT_PROBATION 2 #define RTP_SEQ_MOD (1 << 16) From e65f3c5793e9e2b0a56f4f299a52b6a3cbc5dfea Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 23 May 2019 12:15:26 +0200 Subject: [PATCH 029/339] tests/rtpsession: fix warnings --- .../tests/check/elements/rtpsession.c | 45 ++++++++++++------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 556f34b7583..0abdf538462 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -20,7 +20,9 @@ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ -#define GLIB_DISABLE_DEPRECATION_WARNINGS +#ifndef GLIB_DISABLE_DEPRECATION_WARNINGS +# define GLIB_DISABLE_DEPRECATION_WARNINGS +#endif #include #include @@ -396,7 +398,7 @@ session_harness_advance_and_crank (SessionHarness * h, GstClockTime delta) } static void -session_harness_produce_rtcp (SessionHarness * h, gint num_rtcp_packets) +session_harness_produce_rtcp (SessionHarness * h, guint num_rtcp_packets) { /* due to randomness in rescheduling of RTCP timeout, we need to keep cranking until we have the desired amount of packets */ @@ -505,7 +507,7 @@ GST_START_TEST (test_multiple_ssrc_rr) GstBuffer *in_buf, *out_buf; GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; GstRTCPPacket rtcp_packet; - gint i, j; + guint i, j; guint ssrc_match; guint ssrcs[] = { @@ -567,7 +569,7 @@ GST_START_TEST (test_multiple_senders_roundrobin_rbs) GstBuffer *buf; GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; GstRTCPPacket rtcp_packet; - gint i, j, k; + guint i, j, k; guint32 ssrc; GHashTable *rb_ssrcs, *tmp_set; @@ -926,7 +928,7 @@ typedef struct } RTCPAppResult; static void -on_app_rtcp_cb (GObject * session, guint subtype, guint ssrc, +on_app_rtcp_cb (G_GNUC_UNUSED GObject * session, guint subtype, guint ssrc, const gchar * name, GstBuffer * data, RTCPAppResult * result) { result->subtype = subtype; @@ -993,10 +995,12 @@ GST_START_TEST (test_receive_rtcp_app_packet) GST_END_TEST; static void -stats_test_cb (GObject * object, GParamSpec * spec, gpointer data) +stats_test_cb (G_GNUC_UNUSED GObject * object, G_GNUC_UNUSED GParamSpec * spec, + gpointer data) { guint num_sources = 0; gboolean *cb_called = data; + g_assert (*cb_called == FALSE); /* We should be able to get a rtpsession property @@ -1030,7 +1034,8 @@ GST_START_TEST (test_dont_lock_on_stats) GST_END_TEST; static void -suspicious_bye_cb (GObject * object, GParamSpec * spec, gpointer data) +suspicious_bye_cb (GObject * object, G_GNUC_UNUSED GParamSpec * spec, + gpointer data) { GValueArray *stats_arr; GstStructure *stats, *internal_stats; @@ -1229,7 +1234,8 @@ add_rtcp_sdes_packet (GstBuffer * gstbuf, guint32 ssrc, const char *cname) static void -on_ssrc_collision_cb (GstElement * rtpsession, guint ssrc, gpointer user_data) +on_ssrc_collision_cb (G_GNUC_UNUSED GstElement * rtpsession, + G_GNUC_UNUSED guint ssrc, gpointer user_data) { gboolean *had_collision = user_data; @@ -1875,9 +1881,10 @@ typedef struct } FeedbackRTCPCallbackData; static void -feedback_rtcp_cb (GstElement * element, guint fbtype, guint fmt, - guint sender_ssrc, guint media_ssrc, GstBuffer * fci, - FeedbackRTCPCallbackData * cb_data) +feedback_rtcp_cb (G_GNUC_UNUSED GstElement * element, + G_GNUC_UNUSED guint fbtype, G_GNUC_UNUSED guint fmt, + G_GNUC_UNUSED guint sender_ssrc, G_GNUC_UNUSED guint media_ssrc, + G_GNUC_UNUSED GstBuffer * fci, FeedbackRTCPCallbackData * cb_data) { g_mutex_lock (cb_data->mutex); cb_data->fired = TRUE; @@ -2174,7 +2181,8 @@ typedef struct } BlockingProbeData; static GstPadProbeReturn -on_rtcp_pad_blocked (GstPad * pad, GstPadProbeInfo * info, gpointer user_data) +on_rtcp_pad_blocked (G_GNUC_UNUSED GstPad * pad, + G_GNUC_UNUSED GstPadProbeInfo * info, gpointer user_data) { BlockingProbeData *probe = user_data; @@ -2208,7 +2216,8 @@ session_harness_block_rtcp (SessionHarness * h, BlockingProbeData * probe) } static void -session_harness_unblock_rtcp (SessionHarness * h, BlockingProbeData * probe) +session_harness_unblock_rtcp (G_GNUC_UNUSED SessionHarness * h, + BlockingProbeData * probe) { gst_pad_remove_probe (probe->pad, probe->id); gst_object_unref (probe->pad); @@ -2460,8 +2469,9 @@ GST_START_TEST (test_disable_sr_timestamp) GST_END_TEST; static guint -on_sending_nacks (GObject * internal_session, guint sender_ssrc, - guint media_ssrc, GArray * nacks, GstBuffer * buffer) +on_sending_nacks (G_GNUC_UNUSED GObject * internal_session, + G_GNUC_UNUSED guint sender_ssrc, guint media_ssrc, GArray * nacks, + GstBuffer * buffer) { GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; GstRTCPPacket packet; @@ -2590,7 +2600,8 @@ GST_START_TEST (test_on_sending_nacks) GST_END_TEST; static void -disable_probation_on_new_ssrc (GObject * session, GObject * source) +disable_probation_on_new_ssrc (G_GNUC_UNUSED GObject * session, + GObject * source) { g_object_set (source, "probation", 0, NULL); } @@ -2753,7 +2764,7 @@ test_packet_rate_impl (gboolean stepped) SessionHarness *h = session_harness_new (); GstBuffer *buf; guint i; - const int PROBATION_CNT = 5; + const guint PROBATION_CNT = 5; GstStructure *stats; GObject *source; guint pktrate; From 2240a12e0f5e3f13a37c51879ec2c1af570340b1 Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Thu, 23 May 2019 12:19:13 +0200 Subject: [PATCH 030/339] rtpsession: Add signals for handling profile-specific extension FIXME: Add test for emitting the on-creating-srrr signal when creating early RTCP even though no report blocks have been added. --- .../gst/rtpmanager/rtpsession.c | 97 +++++++++++++++++-- .../gst/rtpmanager/rtpsession.h | 4 + .../tests/check/elements/rtpsession.c | 71 ++++++++++++++ 3 files changed, 165 insertions(+), 7 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index cdc2b4ee048..8aec9de9006 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -45,10 +45,12 @@ enum SIGNAL_ON_SSRC_VALIDATED, SIGNAL_ON_SSRC_ACTIVE, SIGNAL_ON_SSRC_SDES, + SIGNAL_ON_SSRC_PROFILE_SPECIFIC_EXT, SIGNAL_ON_BYE_SSRC, SIGNAL_ON_BYE_TIMEOUT, SIGNAL_ON_TIMEOUT, SIGNAL_ON_SENDER_TIMEOUT, + SIGNAL_ON_CREATING_SR_RR, SIGNAL_ON_SENDING_RTCP, SIGNAL_ON_APP_RTCP, SIGNAL_ON_FEEDBACK_RTCP, @@ -273,6 +275,24 @@ rtp_session_class_init (RTPSessionClass * klass) g_signal_new ("on-ssrc-sdes", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST, G_STRUCT_OFFSET (RTPSessionClass, on_ssrc_sdes), NULL, NULL, NULL, G_TYPE_NONE, 1, RTP_TYPE_SOURCE); + /** + * RTPSession::on-ssrc-profile-specific-ext: + * @session: the object which received the signal + * @src: the RTPSource + * @type: Type of RTCP packet, will be %GST_RTCP_TYPE_SR or + * %GST_RTCP_TYPE_RR + * @pse: a #GstBuffer with the profile-specific extensions data from the + * SR or RR packet + * @rb_count: report block count + * + * Notify that a RTCP profile-specific extension section has been received + */ + rtp_session_signals[SIGNAL_ON_SSRC_PROFILE_SPECIFIC_EXT] = + g_signal_new ("on-ssrc-profile-specific-ext", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, G_STRUCT_OFFSET (RTPSessionClass, on_ssrc_pse), + NULL, NULL, g_cclosure_marshal_generic, + G_TYPE_NONE, 4, RTP_TYPE_SOURCE, G_TYPE_UINT, GST_TYPE_BUFFER, + G_TYPE_UINT); /** * RTPSession::on-bye-ssrc: * @session: the object which received the signal @@ -318,6 +338,21 @@ rtp_session_class_init (RTPSessionClass * klass) G_SIGNAL_RUN_LAST, G_STRUCT_OFFSET (RTPSessionClass, on_sender_timeout), NULL, NULL, NULL, G_TYPE_NONE, 1, RTP_TYPE_SOURCE); + /** + * RTPSession::on-creating-sr-rr + * @session: the object which received the signal + * @src: the RTPSource + * @packet: the #GstRTCPPacket which was just created. + * + * This signal is emitted while creating RTCP SR or RR packet, but before + * sending it. It can be used to add profile-specific extension. + */ + rtp_session_signals[SIGNAL_ON_CREATING_SR_RR] = + g_signal_new ("on-creating-sr-rr", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, G_STRUCT_OFFSET (RTPSessionClass, on_creating_sr_rr), + NULL, NULL, g_cclosure_marshal_generic, G_TYPE_NONE, 2, + RTP_TYPE_SOURCE, G_TYPE_POINTER); + /** * RTPSession::on-sending-rtcp * @session: the object which received the signal @@ -2599,6 +2634,41 @@ rtp_session_process_rb (RTPSession * sess, RTPSource * source, on_ssrc_active (sess, source); } +static void +rtp_session_process_pse (RTPSession * sess, RTPSource * source, + GstRTCPPacket * packet, RTPPacketInfo * pinfo) +{ + guint pse_length, type, rb_count; + guint8 *pse_data; + GstBuffer *pse_buffer; + + if (!gst_rtcp_packet_get_profile_specific_ext (packet, &pse_data, + &pse_length)) + return; + + if (!g_signal_has_handler_pending (sess, + rtp_session_signals[SIGNAL_ON_SSRC_PROFILE_SPECIFIC_EXT], 0, TRUE)) + return; + + GST_DEBUG ("got SR/RR packet with PSE: SSRC %08x, PSE length %u", + source->ssrc, pse_length); + + pse_buffer = gst_buffer_copy_region (packet->rtcp->buffer, + GST_BUFFER_COPY_MEMORY, pse_data - packet->rtcp->map.data, pse_length); + GST_BUFFER_TIMESTAMP (pse_buffer) = pinfo->running_time; + type = gst_rtcp_packet_get_type (packet); + rb_count = gst_rtcp_packet_get_rb_count (packet); + + g_object_ref (source); + RTP_SESSION_UNLOCK (sess); + g_signal_emit (sess, rtp_session_signals[SIGNAL_ON_SSRC_PROFILE_SPECIFIC_EXT], + 0, source, type, pse_buffer, rb_count); + RTP_SESSION_LOCK (sess); + g_object_unref (source); + + gst_buffer_unref (pse_buffer); +} + /* A Sender report contains statistics about how the sender is doing. This * includes timing informataion such as the relation between RTP and NTP * timestamps and the number of packets/bytes it sent to us. @@ -2649,6 +2719,7 @@ rtp_session_process_sr (RTPSession * sess, GstRTCPPacket * packet, on_new_ssrc (sess, source); rtp_session_process_rb (sess, source, packet, pinfo); + rtp_session_process_pse (sess, source, packet, pinfo); out: g_object_unref (source); @@ -2684,6 +2755,7 @@ rtp_session_process_rr (RTPSession * sess, GstRTCPPacket * packet, on_new_ssrc (sess, source); rtp_session_process_rb (sess, source, packet, pinfo); + rtp_session_process_pse (sess, source, packet, pinfo); out: g_object_unref (source); @@ -3965,7 +4037,7 @@ typedef struct gboolean timeout_inactive_sources; } ReportData; -static void +static gboolean session_start_rtcp (RTPSession * sess, ReportData * data) { GstRTCPPacket *packet = &data->packet; @@ -3977,6 +4049,9 @@ session_start_rtcp (RTPSession * sess, ReportData * data) gst_rtcp_buffer_map (data->rtcp, GST_MAP_READWRITE, rtcp); + if (data->is_early && sess->reduced_size_rtcp) + return FALSE; + if (RTP_SOURCE_IS_SENDER (own) && (!data->is_early || !sess->reduced_size_rtcp || sess->sr_req_pending)) { guint64 ntptime; @@ -4007,6 +4082,8 @@ session_start_rtcp (RTPSession * sess, ReportData * data) gst_rtcp_buffer_add_packet (rtcp, GST_RTCP_TYPE_RR, packet); gst_rtcp_packet_rr_set_ssrc (packet, own->ssrc); } + + return TRUE; } /* construct a Sender or Receiver Report */ @@ -4674,6 +4751,7 @@ generate_rtcp (const gchar * key, RTPSource * source, ReportData * data) gboolean is_bye = FALSE; ReportOutput *output; gboolean sr_req_pending = sess->sr_req_pending; + gboolean is_sr_rr = FALSE; /* only generate RTCP for active internal sources */ if (!source->internal || source->sent_bye) @@ -4692,17 +4770,22 @@ generate_rtcp (const gchar * key, RTPSource * source, ReportData * data) data->source = source; /* open packet */ - session_start_rtcp (sess, data); + is_sr_rr = session_start_rtcp (sess, data); if (source->marked_bye) { /* send BYE */ make_source_bye (sess, source, data); is_bye = TRUE; - } else if (!data->is_early) { - /* loop over all known sources and add report blocks. If we are early, we - * just make a minimal RTCP packet and skip this step */ - g_hash_table_foreach (sess->ssrcs[sess->mask_idx], - (GHFunc) session_report_blocks, data); + } else if (is_sr_rr) { + if (!data->is_early) { + /* loop over all known sources and add report blocks. If we are early, we + * just make a minimal RTCP packet and skip this step */ + g_hash_table_foreach (sess->ssrcs[sess->mask_idx], + (GHFunc) session_report_blocks, data); + } + /* Optionally add profile-specific extension */ + g_signal_emit (sess, rtp_session_signals[SIGNAL_ON_CREATING_SR_RR], 0, + source, &data->packet); } if (!data->has_sdes && (!data->is_early || !sess->reduced_size_rtcp || sr_req_pending)) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h index aef60260f9a..aefd0b76a47 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h @@ -345,6 +345,8 @@ struct _RTPSessionClass { void (*on_ssrc_validated) (RTPSession *sess, RTPSource *source); void (*on_ssrc_active) (RTPSession *sess, RTPSource *source); void (*on_ssrc_sdes) (RTPSession *sess, RTPSource *source); + void (*on_ssrc_pse) (RTPSession *sess, RTPSource *source, + guint type, GstBuffer *pse, guint rb_count); void (*on_bye_ssrc) (RTPSession *sess, RTPSource *source); void (*on_bye_timeout) (RTPSession *sess, RTPSource *source); void (*on_timeout) (RTPSession *sess, RTPSource *source); @@ -353,6 +355,8 @@ struct _RTPSessionClass { gboolean early); void (*on_app_rtcp) (RTPSession *sess, guint subtype, guint ssrc, const gchar *name, GstBuffer *data); + void (*on_creating_sr_rr) (RTPSession *sess, guint type, RTPSource *source, + GstRTCPPacket * packet); void (*on_feedback_rtcp) (RTPSession *sess, guint type, guint fbtype, guint sender_ssrc, guint media_ssrc, GstBuffer *fci); gboolean (*send_rtcp) (RTPSession *sess, GstClockTime max_delay); diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 0abdf538462..6caf0f61ed3 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -2819,6 +2819,76 @@ GST_START_TEST (test_stepped_packet_rate) GST_END_TEST; +static const guint8 pse_data[] = { + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77 +}; + +static void +_creating_srrr (G_GNUC_UNUSED GObject * session, G_GNUC_UNUSED GObject * src, + GstRTCPPacket * packet) +{ + fail_unless_equals_int (GST_RTCP_TYPE_RR, gst_rtcp_packet_get_type (packet)); + gst_rtcp_packet_add_profile_specific_ext (packet, pse_data, + sizeof (pse_data)); +} + +static void +_ssrc_pse (G_GNUC_UNUSED GObject * session, GObject * src, guint type, + GstBuffer * pse, G_GNUC_UNUSED guint rb_count, gpointer userdata) +{ + g_object_get (src, "ssrc", userdata, NULL); + + fail_unless_equals_int (GST_RTCP_TYPE_RR, type); + fail_unless_equals_int (0, + gst_buffer_memcmp (pse, 0, pse_data, sizeof (pse_data))); +} + +GST_START_TEST (test_creating_srrr) +{ + SessionHarness *h = session_harness_new (); + guint i; + GstFlowReturn res; + GstBuffer *buf; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + GstRTCPPacket packet; + guint pse_ssrc = 0; + + /* Connect to on-creating-sr-rr which will append 8 bytes of + * profile-specific extension data */ + g_object_set (h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); + g_signal_connect (h->internal_session, "on-creating-sr-rr", + G_CALLBACK (_creating_srrr), NULL); + + /* receive some buffers */ + for (i = 0; i < 2; i++) { + buf = generate_test_buffer (i, 0x01BADBAD); + res = session_harness_recv_rtp (h, buf); + fail_unless_equals_int (GST_FLOW_OK, res); + } + + session_harness_produce_rtcp (h, 1); + buf = session_harness_pull_rtcp (h); + fail_unless (gst_rtcp_buffer_validate (buf)); + gst_rtcp_buffer_map (buf, GST_MAP_READ, &rtcp); + fail_unless (gst_rtcp_buffer_get_first_packet (&rtcp, &packet)); + fail_unless_equals_int (GST_RTCP_TYPE_RR, gst_rtcp_packet_get_type (&packet)); + fail_unless_equals_int (2, /* 2x 32bit words */ + gst_rtcp_packet_get_profile_specific_ext_length (&packet)); + gst_rtcp_buffer_unmap (&rtcp); + + /* now "receive" the same RTCP buffer into the session to test + * on-ssrc-profile-specific-ext signal */ + g_signal_connect (h->internal_session, "on-ssrc-profile-specific-ext", + G_CALLBACK (_ssrc_pse), &pse_ssrc); + fail_unless_equals_int (0, pse_ssrc); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtcp (h, buf)); + fail_unless_equals_int (0xDEADBEEF, pse_ssrc); + + session_harness_free (h); +} + +GST_END_TEST; + /********************* TWCC-tests *********************/ static GstRTCPFBType @@ -5192,6 +5262,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_clear_pt_map_stress); tcase_add_test (tc_chain, test_packet_rate); tcase_add_test (tc_chain, test_stepped_packet_rate); + tcase_add_test (tc_chain, test_creating_srrr); /* twcc */ tcase_add_loop_test (tc_chain, test_twcc_header_and_run_length, From d17e358c6c48a12673542dc5c593d342e0a1c518 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 25 Aug 2017 11:58:38 +0200 Subject: [PATCH 031/339] rtpsession: Allow instant transmission of RTCP packets By specifying max_delay of 0, it effectively bypasses the RTCP throttling mechanism, and tries to get an RTCP packet on the wire as quickly as possible. --- .../gst/rtpmanager/rtpsession.c | 17 +++++++--- .../tests/check/elements/rtpsession.c | 32 +++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 8aec9de9006..16c88120a8d 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -4644,7 +4644,6 @@ is_rtcp_time (RTPSession * sess, GstClockTime current_time, ReportData * data) /* If this is the first RTCP packet, we can reconsider anything based * on the last RTCP send time because there was none. */ - g_warn_if_fail (!data->is_early); data->is_early = FALSE; new_send_time = current_time; } @@ -5150,7 +5149,7 @@ gboolean rtp_session_request_early_rtcp (RTPSession * sess, GstClockTime current_time, GstClockTime max_delay) { - GstClockTime T_dither_max, T_rr, offset = 0; + GstClockTime T_dither_max = 0, T_rr, offset = 0; gboolean ret; gboolean allow_early; @@ -5162,6 +5161,14 @@ rtp_session_request_early_rtcp (RTPSession * sess, GstClockTime current_time, * to be sent */ sess->rtp_profile = GST_RTP_PROFILE_AVPF; + /* special case, a max_delay of 0 means a RTCP-packet should be forced out + instantly, effectively ignoring RFC 4585 section 3.5. Usecases includes + some non-standard ways of using RTCP, like towards Microsoft Lync */ + if (max_delay == 0) { + sess->next_rtcp_check_time = current_time; + goto send_early; + } + /* Check if already requested */ /* RFC 4585 section 3.5.2 step 2 */ if (GST_CLOCK_TIME_IS_VALID (sess->next_early_rtcp_time)) { @@ -5209,9 +5216,7 @@ rtp_session_request_early_rtcp (RTPSession * sess, GstClockTime current_time, /* When there is one auxiliary stream the session can still do point * to point. */ - if (sess->is_doing_ptp) { - T_dither_max = 0; - } else { + if (!sess->is_doing_ptp) { /* Divide by 2 because l = 0.5 */ T_dither_max = T_rr; T_dither_max /= 2; @@ -5274,6 +5279,8 @@ rtp_session_request_early_rtcp (RTPSession * sess, GstClockTime current_time, goto end; } +send_early: + /* RFC 4585 section 3.5.2 step 4b */ if (T_dither_max) { /* Schedule an early transmission later */ diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 6caf0f61ed3..3f4887d5da3 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -5219,6 +5219,37 @@ GST_START_TEST (test_twcc_sent_packets_wrap) GST_END_TEST; +GST_START_TEST (test_send_rtcp_instantly) +{ + SessionHarness *h = session_harness_new (); + gboolean ret; + const GstClockTime now = 123456789; + + /* advance the clock to "now" */ + gst_test_clock_set_time (h->testclock, now); + + /* verify the RTCP thread has not started */ + fail_unless_equals_int (0, gst_test_clock_peek_id_count (h->testclock)); + /* and that no RTCP has been pushed */ + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h->rtcp_h)); + + /* then ask explicitly to send RTCP with 0 timeout (now!) */ + g_signal_emit_by_name (h->internal_session, "send-rtcp-full", 0, &ret); + /* this is TRUE due to ? */ + fail_unless (ret == TRUE); + + /* "crank" and verify RTCP now was sent */ + session_harness_crank_clock (h); + gst_buffer_unref (session_harness_pull_rtcp (h)); + + /* and check the time is "now" */ + fail_unless_equals_int64 (now, gst_clock_get_time (GST_CLOCK (h->testclock))); + + session_harness_free (h); +} + +GST_END_TEST; + static Suite * rtpsession_suite (void) { @@ -5313,6 +5344,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_twcc_overwrites_exthdr_seqnum_if_present); tcase_add_test (tc_chain, test_twcc_sent_packets_wrap); + tcase_add_test (tc_chain, test_send_rtcp_instantly); return s; } From 15767de28d1f01fbe7004e576f0496e958668dd2 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 24 May 2016 16:50:41 +0200 Subject: [PATCH 032/339] rtpsession: Add "send-bye" action-signal --- .../gst/rtpmanager/gstrtpsession.c | 32 +++++++++--- .../gst/rtpmanager/gstrtpsession.h | 1 + .../tests/check/elements/rtpsession.c | 51 +++++++++++++++++++ 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c index c50a8072968..7be9c202b92 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c @@ -193,6 +193,7 @@ enum { SIGNAL_REQUEST_PT_MAP, SIGNAL_CLEAR_PT_MAP, + SIGNAL_SEND_BYE, SIGNAL_ON_NEW_SSRC, SIGNAL_ON_SSRC_COLLISION, @@ -364,6 +365,7 @@ static gboolean gst_rtp_session_setcaps_send_rtp (GstPad * pad, GstRtpSession * rtpsession, GstCaps * caps); static void gst_rtp_session_clear_pt_map (GstRtpSession * rtpsession); +static void gst_rtp_session_send_bye (GstRtpSession * rtpsession); static GstStructure *gst_rtp_session_create_stats (GstRtpSession * rtpsession); @@ -537,6 +539,18 @@ gst_rtp_session_class_init (GstRtpSessionClass * klass) G_STRUCT_OFFSET (GstRtpSessionClass, clear_pt_map), NULL, NULL, NULL, G_TYPE_NONE, 0, G_TYPE_NONE); + /** + * GstRtpSession::send-bye: + * @sess: the object which received the signal + * + * Send BYE on all active sources in the session. + */ + gst_rtp_session_signals[SIGNAL_SEND_BYE] = + g_signal_new ("send-bye", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, + G_STRUCT_OFFSET (GstRtpSessionClass, send_bye), + NULL, NULL, g_cclosure_marshal_generic, G_TYPE_NONE, 0, G_TYPE_NONE); + /** * GstRtpSession::on-new-ssrc: * @sess: the object which received the signal @@ -856,6 +870,7 @@ gst_rtp_session_class_init (GstRtpSessionClass * klass) GST_DEBUG_FUNCPTR (gst_rtp_session_release_pad); klass->clear_pt_map = GST_DEBUG_FUNCPTR (gst_rtp_session_clear_pt_map); + klass->send_bye = GST_DEBUG_FUNCPTR (gst_rtp_session_send_bye); /* sink pads */ gst_element_class_add_static_pad_template (gstelement_class, @@ -1427,6 +1442,15 @@ gst_rtp_session_clear_pt_map (GstRtpSession * rtpsession) GST_RTP_SESSION_UNLOCK (rtpsession); } +static void +gst_rtp_session_send_bye (GstRtpSession * rtpsession) +{ + GstClockTime current_time = gst_clock_get_time (rtpsession->priv->sysclock); + GST_DEBUG_OBJECT (rtpsession, "scheduling BYE message"); + rtp_session_mark_all_bye (rtpsession->priv->session, "End Of Stream"); + rtp_session_schedule_bye (rtpsession->priv->session, current_time); +} + /* called when the session manager has an RTP packet ready to be pushed */ static GstFlowReturn gst_rtp_session_process_rtp (RTPSession * sess, RTPSource * src, @@ -2287,19 +2311,15 @@ gst_rtp_session_event_send_rtp_sink (GstPad * pad, GstObject * parent, break; } case GST_EVENT_EOS:{ - GstClockTime current_time; - /* push downstream FIXME, we are not supposed to leave the session just * because we stop sending. */ ret = gst_pad_push_event (rtpsession->send_rtp_src, event); - current_time = gst_clock_get_time (rtpsession->priv->sysclock); + GST_RTP_SESSION_LOCK (rtpsession); rtpsession->priv->send_rtp_sink_eos = TRUE; GST_RTP_SESSION_UNLOCK (rtpsession); - GST_DEBUG_OBJECT (rtpsession, "scheduling BYE message"); - rtp_session_mark_all_bye (rtpsession->priv->session, "End Of Stream"); - rtp_session_schedule_bye (rtpsession->priv->session, current_time); + gst_rtp_session_send_bye (rtpsession); break; } default:{ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.h index 5a3ecfc1c17..d40c9e801d2 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.h @@ -64,6 +64,7 @@ struct _GstRtpSessionClass { /* signals */ GstCaps* (*request_pt_map) (GstRtpSession *sess, guint pt); void (*clear_pt_map) (GstRtpSession *sess); + void (*send_bye) (GstRtpSession *sess); void (*on_new_ssrc) (GstRtpSession *sess, guint32 ssrc); void (*on_ssrc_collision) (GstRtpSession *sess, guint32 ssrc); diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 3f4887d5da3..a3e4f5db2c1 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -5250,6 +5250,56 @@ GST_START_TEST (test_send_rtcp_instantly) GST_END_TEST; +GST_START_TEST (test_send_bye_signal) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + GstRTCPPacket rtcp_packet; + guint32 ssrc; + + /* push a buffer to establish an internal source */ + fail_unless_equals_int (GST_FLOW_OK, + session_harness_send_rtp (h, generate_test_buffer (0, 0xDEADBEEF))); + + /* emit the signal to signal bye on all sources */ + g_signal_emit_by_name (h->session, "send-bye"); + + session_harness_produce_rtcp (h, 1); + buf = session_harness_pull_rtcp (h); + + fail_unless (gst_rtcp_buffer_validate (buf)); + gst_rtcp_buffer_map (buf, GST_MAP_READ, &rtcp); + /* our RTCP buffer has 3 packets */ + fail_unless_equals_int (3, gst_rtcp_buffer_get_packet_count (&rtcp)); + + /* first a Sender Report */ + fail_unless (gst_rtcp_buffer_get_first_packet (&rtcp, &rtcp_packet)); + fail_unless_equals_int (GST_RTCP_TYPE_SR, + gst_rtcp_packet_get_type (&rtcp_packet)); + gst_rtcp_packet_sr_get_sender_info (&rtcp_packet, &ssrc, NULL, NULL, + NULL, NULL); + fail_unless_equals_int (0xDEADBEEF, ssrc); + fail_unless (gst_rtcp_packet_move_to_next (&rtcp_packet)); + + /* then a SDES */ + fail_unless_equals_int (GST_RTCP_TYPE_SDES, + gst_rtcp_packet_get_type (&rtcp_packet)); + fail_unless_equals_int (0xDEADBEEF, + gst_rtcp_packet_sdes_get_ssrc (&rtcp_packet)); + fail_unless (gst_rtcp_packet_move_to_next (&rtcp_packet)); + + /* and finally the BYE we asked for */ + fail_unless_equals_int (GST_RTCP_TYPE_BYE, + gst_rtcp_packet_get_type (&rtcp_packet)); + + gst_rtcp_buffer_unmap (&rtcp); + gst_buffer_unref (buf); + session_harness_free (h); +} + +GST_END_TEST; + static Suite * rtpsession_suite (void) { @@ -5345,6 +5395,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_twcc_sent_packets_wrap); tcase_add_test (tc_chain, test_send_rtcp_instantly); + tcase_add_test (tc_chain, test_send_bye_signal); return s; } From 275473ff6d9afc95d1f4f42d2674e9725849a688 Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Thu, 23 May 2019 14:45:50 +0200 Subject: [PATCH 033/339] rtpsession: process RB by internal sources [CHECKME!!! DOES NOT WORK] When we deal with receiver reports containing information about multiple internal SSRCs we used to save data only for the SSRC in the last report block. Thus the stats would never reflect receiver reports of other SSRCs. For example. Non internal source 0xAAAA receives RR with 3 RBs about internal sources 0x1111, 0x2222 and 0x3333. Because 0x3333 was the last processed RB we will never see the receiver report of 0x1111 and 0x2222 in the RTP session stats. With this patch we save receiver report information in the corresponding internal source. --- .../gst/rtpmanager/rtpsession.c | 2 +- .../tests/check/elements/rtpsession.c | 94 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 16c88120a8d..e56afeb7f0c 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -2627,7 +2627,7 @@ rtp_session_process_rb (RTPSession * sess, RTPSource * source, * the sender of the RTCP message. We could also compare our stats against * the other sender to see if we are better or worse. */ /* FIXME, need to keep track who the RB block is from */ - rtp_source_process_rb (source, ssrc, pinfo->ntpnstime, fractionlost, + rtp_source_process_rb (src, ssrc, pinfo->ntpnstime, fractionlost, packetslost, exthighestseq, jitter, lsr, dlsr); } } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index a3e4f5db2c1..69b47ed1340 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -5300,6 +5300,99 @@ GST_START_TEST (test_send_bye_signal) GST_END_TEST; +GST_START_TEST (test_stats_rtcp_with_multiple_rb) +{ + SessionHarness *h = session_harness_new (); + guint j, k; + GstFlowReturn res; + GstRTCPPacket packet; + gint internal_stats_entries; + GstBuffer *buf = NULL; + GstStructure *stats = NULL; + GValueArray *source_stats = NULL; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + + /* Push RTP from our send SSRCs */ + for (j = 0; j < 5; j++) { /* packets per ssrc */ + for (k = 0; k < 2; k++) { /* number of ssrcs */ + buf = generate_test_buffer (j, 10000 + k); + res = session_harness_send_rtp (h, buf); + fail_unless_equals_int (GST_FLOW_OK, res); + } + } + + /* Push RTCP RR with 2 RBs corresponding to our send SSRCs */ + buf = gst_rtcp_buffer_new (1000); + fail_unless (gst_rtcp_buffer_map (buf, GST_MAP_READWRITE, &rtcp)); + fail_unless (gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_RR, &packet)); + gst_rtcp_packet_rr_set_ssrc (&packet, 20000); + for (k = 0; k < 2; k++) { + guint32 ssrc = 10000 + k; + guint32 jitter = (ssrc % 2) + 10; + gst_rtcp_packet_add_rb (&packet, 10000 + k, /* ssrc */ + 0, /* fractionlost */ + 0, /* packetslost */ + 4, /* exthighestseq */ + jitter, /* jitter */ + 0, /* lsr */ + 0); /* dlsr */ + } + gst_rtcp_buffer_unmap (&rtcp); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtcp (h, buf)); + + /* Check that the stats reflect the data we received in RTCP */ + internal_stats_entries = 0; + g_object_get (h->session, "stats", &stats, NULL); + fail_unless (stats != NULL); + source_stats = + g_value_get_boxed (gst_structure_get_value (stats, "source-stats")); + fail_unless (source_stats != NULL); + for (j = 0; j < source_stats->n_values; j++) { + guint32 ssrc; + gboolean internal; + GstStructure *s = + g_value_get_boxed (g_value_array_get_nth (source_stats, j)); + fail_unless (gst_structure_get (s, "internal", G_TYPE_BOOLEAN, &internal, + "ssrc", G_TYPE_UINT, &ssrc, NULL)); + if (internal) { + gboolean have_rb; + guint rb_fractionlost; + gint rb_packetslost; + guint rb_exthighestseq; + guint rb_jitter; + guint rb_lsr; + guint rb_dlsr; + + fail_unless_equals_int (ssrc, 10000 + internal_stats_entries); + + fail_unless (gst_structure_get (s, + "have-rb", G_TYPE_BOOLEAN, &have_rb, + "rb-fractionlost", G_TYPE_UINT, &rb_fractionlost, + "rb-packetslost", G_TYPE_INT, &rb_packetslost, + "rb-exthighestseq", G_TYPE_UINT, &rb_exthighestseq, + "rb-jitter", G_TYPE_UINT, &rb_jitter, + "rb-lsr", G_TYPE_UINT, &rb_lsr, + "rb-dlsr", G_TYPE_UINT, &rb_dlsr, NULL)); + fail_unless (have_rb); + fail_unless_equals_int (rb_fractionlost, 0); + fail_unless_equals_int (rb_packetslost, 0); + fail_unless_equals_int (rb_exthighestseq, 4); + fail_unless_equals_int (rb_jitter, (ssrc % 2) + 10); + fail_unless_equals_int (rb_lsr, 0); + fail_unless_equals_int (rb_dlsr, 0); + internal_stats_entries++; + } else { + fail_unless_equals_int (ssrc, 20000); + } + } + fail_unless_equals_int (internal_stats_entries, 2); + + gst_structure_free (stats); + session_harness_free (h); +} + +GST_END_TEST; + static Suite * rtpsession_suite (void) { @@ -5396,6 +5489,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_send_rtcp_instantly); tcase_add_test (tc_chain, test_send_bye_signal); + tcase_skip_broken_test (tc_chain, test_stats_rtcp_with_multiple_rb); return s; } From 2b7b9d7e6f80505c8ae3083c32baa74508ed942c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Wed, 27 Sep 2023 11:17:39 +0200 Subject: [PATCH 034/339] rtpsession: Add property "stats-min-interval" and notify "stats" Notify about changed stats when RTCP is sent. Since notifying on every RTCP packet (when early RTCP is used) may introduce significant overhead, so the property "stats-min-interval" is added in order to control how often this notification can happen. --- .../gst/rtpmanager/rtpsession.c | 25 ++++++- .../gst/rtpmanager/rtpsession.h | 2 + .../tests/check/elements/rtpsession.c | 66 +++++++++++++++++++ 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index e56afeb7f0c..1fc96ad0390 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -86,6 +86,7 @@ enum #define DEFAULT_TWCC_FEEDBACK_INTERVAL GST_CLOCK_TIME_NONE #define DEFAULT_UPDATE_NTP64_HEADER_EXT TRUE #define DEFAULT_TIMEOUT_INACTIVE_SOURCES TRUE +#define DEFAULT_STATS_NOTIFY_MIN_INTERVAL 0 enum { @@ -109,6 +110,7 @@ enum PROP_MAX_DROPOUT_TIME, PROP_MAX_MISORDER_TIME, PROP_STATS, + PROP_STATS_NOTIFY_MIN_INTERVAL, PROP_RTP_PROFILE, PROP_RTCP_REDUCED_SIZE, PROP_RTCP_DISABLE_SR_TIMESTAMP, @@ -674,6 +676,13 @@ rtp_session_class_init (RTPSessionClass * klass) "Various statistics", GST_TYPE_STRUCTURE, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS); + properties[PROP_STATS_NOTIFY_MIN_INTERVAL] = + g_param_spec_uint ("stats-notify-min-interval", + "Stats notify minimum interval", + "Minimum interval between emitting notify signal for stats (in ms)", + 0, G_MAXUINT, DEFAULT_STATS_NOTIFY_MIN_INTERVAL, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + properties[PROP_RTP_PROFILE] = g_param_spec_enum ("rtp-profile", "RTP Profile", "RTP profile to use for this session", GST_TYPE_RTP_PROFILE, @@ -847,6 +856,9 @@ rtp_session_init (RTPSession * sess) sess->reduced_size_rtcp = DEFAULT_RTCP_REDUCED_SIZE; sess->timestamp_sender_reports = !DEFAULT_RTCP_DISABLE_SR_TIMESTAMP; + sess->last_stats_notify_time = GST_CLOCK_TIME_NONE; + sess->stats_notify_min_interval_ms = DEFAULT_STATS_NOTIFY_MIN_INTERVAL; + sess->is_doing_ptp = TRUE; sess->twcc = rtp_twcc_manager_new (sess->mtu); @@ -1047,6 +1059,9 @@ rtp_session_set_property (GObject * object, guint prop_id, case PROP_MAX_MISORDER_TIME: sess->max_misorder_time = g_value_get_uint (value); break; + case PROP_STATS_NOTIFY_MIN_INTERVAL: + sess->stats_notify_min_interval_ms = g_value_get_uint (value); + break; case PROP_RTP_PROFILE: sess->rtp_profile = g_value_get_enum (value); /* trigger reconsideration */ @@ -1156,6 +1171,9 @@ rtp_session_get_property (GObject * object, guint prop_id, case PROP_STATS: g_value_take_boxed (value, rtp_session_create_stats (sess)); break; + case PROP_STATS_NOTIFY_MIN_INTERVAL: + g_value_set_uint (value, sess->stats_notify_min_interval_ms); + break; case PROP_RTP_PROFILE: g_value_set_enum (value, sess->rtp_profile); break; @@ -5070,7 +5088,12 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, /* notify about updated statistics */ if (!twcc_only) { - g_object_notify_by_pspec (G_OBJECT (sess), properties[PROP_STATS]); + if (!GST_CLOCK_TIME_IS_VALID (sess->last_stats_notify_time) || + (GST_CLOCK_DIFF (sess->last_stats_notify_time, current_time) >= + sess->stats_notify_min_interval_ms * GST_MSECOND)) { + g_object_notify_by_pspec (G_OBJECT (sess), properties[PROP_STATS]); + sess->last_stats_notify_time = current_time; + } } /* push out the RTCP packets */ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h index aefd0b76a47..3c07a017974 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h @@ -300,6 +300,8 @@ struct _RTPSession { RTPSessionStats stats; RTPSessionStats bye_stats; + GstClockTime last_stats_notify_time; + guint stats_notify_min_interval_ms; gboolean favor_new; GstClockTime rtcp_feedback_retention_window; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 69b47ed1340..7ee26de811a 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -4,6 +4,7 @@ * * Copyright (C) <2009> Wim Taymans * Copyright (C) 2013 Collabora Ltd. + * Copyright (C) <2018> Havard Graff * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -5393,6 +5394,70 @@ GST_START_TEST (test_stats_rtcp_with_multiple_rb) GST_END_TEST; +static void +count_report_stats (G_GNUC_UNUSED GObject * object, + G_GNUC_UNUSED GParamSpec * spec, gint * counter) +{ + *counter += 1; +} + +static void +on_sending_rtcp_add_new_if_empty (G_GNUC_UNUSED GObject * rtpsession, + GstBuffer * rtcp_buffer, G_GNUC_UNUSED gboolean is_early, + G_GNUC_UNUSED gpointer user_data) +{ + if (gst_buffer_get_size (rtcp_buffer) == 0) { + GstRTCPPacket packet; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + + gst_rtcp_buffer_map (rtcp_buffer, GST_MAP_READWRITE, &rtcp); + + gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_APP, &packet); + gst_rtcp_packet_app_set_subtype (&packet, 1); + gst_rtcp_packet_app_set_ssrc (&packet, 0x12345678); + gst_rtcp_packet_app_set_name (&packet, "foo"); + + gst_rtcp_buffer_unmap (&rtcp); + } +} + +GST_START_TEST (test_report_stats_only_on_regular_rtcp) +{ + SessionHarness *h = session_harness_new (); + gint stats_callback_count = 0; + gint i; + + g_object_set (h->internal_session, "probation", 1, "rtcp-reduced-size", TRUE, + "stats-notify-min-interval", 3000, NULL); + g_signal_connect (h->session, "notify::stats", + G_CALLBACK (count_report_stats), &stats_callback_count); + + /* Not allowed to send empty packets, so need to add feedback */ + g_signal_connect (h->internal_session, "on-sending-rtcp", + G_CALLBACK (on_sending_rtcp_add_new_if_empty), NULL); + + fail_unless_equals_int (GST_FLOW_OK, + session_harness_recv_rtp (h, generate_test_buffer (0, 0x12345678))); + + session_harness_produce_rtcp (h, 1); + gst_buffer_unref (session_harness_pull_rtcp (h)); + fail_unless_equals_int (stats_callback_count, 1); + + /* send 10 rtcp-packets that should *not* generate stats */ + for (i = 0; i < 10; i++) { + gboolean ret; + g_signal_emit_by_name (h->internal_session, "send-rtcp-full", 0, &ret); + session_harness_advance_and_crank (h, 10 * GST_MSECOND); + } + + /* verify we have generated less than 3 stats for all these packets */ + fail_unless (stats_callback_count < 3); + + session_harness_free (h); +} + +GST_END_TEST; + static Suite * rtpsession_suite (void) { @@ -5490,6 +5555,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_send_rtcp_instantly); tcase_add_test (tc_chain, test_send_bye_signal); tcase_skip_broken_test (tc_chain, test_stats_rtcp_with_multiple_rb); + tcase_add_test (tc_chain, test_report_stats_only_on_regular_rtcp); return s; } From ade788a0450f2896dc08dfcfc72d2fee2e744790 Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Fri, 18 Jul 2014 14:19:12 +0200 Subject: [PATCH 035/339] rtpsession: Add ssrc-collision-detection property [ADD TEST] Added to make it possible to disable detection. Still enabled by default. --- .../gst/rtpmanager/rtpsession.c | 24 +++++++++++++++---- .../gst/rtpmanager/rtpsession.h | 1 + 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 1fc96ad0390..fec57bec388 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -79,6 +79,7 @@ enum #define DEFAULT_PROBATION RTP_DEFAULT_PROBATION #define DEFAULT_MAX_DROPOUT_TIME 60000 #define DEFAULT_MAX_MISORDER_TIME 2000 +#define DEFAULT_SSRC_COLLISION_DETECTION TRUE #define DEFAULT_RTP_PROFILE GST_RTP_PROFILE_AVP #define DEFAULT_RTCP_REDUCED_SIZE FALSE #define DEFAULT_RTCP_DISABLE_SR_TIMESTAMP FALSE @@ -109,6 +110,7 @@ enum PROP_PROBATION, PROP_MAX_DROPOUT_TIME, PROP_MAX_MISORDER_TIME, + PROP_SSRC_COLLISION_DETECTION, PROP_STATS, PROP_STATS_NOTIFY_MIN_INTERVAL, PROP_RTP_PROFILE, @@ -656,6 +658,13 @@ rtp_session_class_init (RTPSessionClass * klass) 0, G_MAXUINT, DEFAULT_MAX_MISORDER_TIME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + properties[PROP_SSRC_COLLISION_DETECTION] = + g_param_spec_boolean ("ssrc-collision-detection", + "SSRC collision detection", + "Enable/Disable SSRC collision detection for remote sources", + DEFAULT_SSRC_COLLISION_DETECTION, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + /** * RTPSession:stats: * @@ -819,6 +828,7 @@ rtp_session_init (RTPSession * sess) sess->max_dropout_time = DEFAULT_MAX_DROPOUT_TIME; sess->max_misorder_time = DEFAULT_MAX_MISORDER_TIME; sess->favor_new = DEFAULT_FAVOR_NEW; + sess->ssrc_collision_detection = DEFAULT_SSRC_COLLISION_DETECTION; /* some default SDES entries */ sess->sdes = gst_structure_new_empty ("application/x-rtp-source-sdes"); @@ -1062,6 +1072,9 @@ rtp_session_set_property (GObject * object, guint prop_id, case PROP_STATS_NOTIFY_MIN_INTERVAL: sess->stats_notify_min_interval_ms = g_value_get_uint (value); break; + case PROP_SSRC_COLLISION_DETECTION: + sess->ssrc_collision_detection = g_value_get_boolean (value); + break; case PROP_RTP_PROFILE: sess->rtp_profile = g_value_get_enum (value); /* trigger reconsideration */ @@ -1168,6 +1181,9 @@ rtp_session_get_property (GObject * object, guint prop_id, case PROP_MAX_MISORDER_TIME: g_value_set_uint (value, sess->max_misorder_time); break; + case PROP_SSRC_COLLISION_DETECTION: + g_value_set_boolean (value, sess->ssrc_collision_detection); + break; case PROP_STATS: g_value_take_boxed (value, rtp_session_create_stats (sess)); break; @@ -1865,11 +1881,11 @@ check_collision (RTPSession * sess, RTPSource * source, gchar *buf1; buf1 = __g_socket_address_to_string (pinfo->address); - GST_LOG ("Known conflict on %x for %s, dropping packet", ssrc, - buf1); + GST_LOG ("Known conflict on %x for %s, %s packet", ssrc, + buf1, sess->ssrc_collision_detection ? "dropping" : "ignoring"); g_free (buf1); - return TRUE; + return sess->ssrc_collision_detection; } else { gchar *buf1, *buf2; @@ -1897,7 +1913,7 @@ check_collision (RTPSession * sess, RTPSource * source, } } else { /* Don't need to save old addresses, we ignore new sources */ - return TRUE; + return sess->ssrc_collision_detection; /* if enabled */ } } } else { diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h index 3c07a017974..405ea6a7297 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h @@ -246,6 +246,7 @@ struct _RTPSession { guint probation; guint32 max_dropout_time; guint32 max_misorder_time; + gboolean ssrc_collision_detection; GstRTPProfile rtp_profile; From a9f96db29c4f51e3567aad9fbda21d63f4d8c00a Mon Sep 17 00:00:00 2001 From: John Bassett Date: Thu, 14 Jan 2016 17:00:42 +0000 Subject: [PATCH 036/339] rtpsession: Generate regular SR/RR when lots of early RTCP [ADD TEST] If an application generates a steady stream of early RTCP using the 'send-rtcp' signal to rtpsession it was possible that the normal sending of Sender/Receiver reports could be inhibited. This patch ensures that SR/RR are still sent regularly. --- .../gst/rtpmanager/rtpsession.c | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index fec57bec388..6e0ef93d953 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -4711,6 +4711,26 @@ is_rtcp_time (RTPSession * sess, GstClockTime current_time, ReportData * data) } } sess->next_rtcp_check_time = current_time + interval; + } else { + /* + * Even though this is an early request if we haven't sent a regular packet + * for at least 'interval' then we need to include an RR/SR + */ + if (sess->last_rtcp_send_time != GST_CLOCK_TIME_NONE + && interval != GST_CLOCK_TIME_NONE) { + GstClockTime elapsed = current_time - sess->last_rtcp_send_time; + + if (elapsed > interval) { + data->is_early = FALSE; + GST_DEBUG_OBJECT (sess, + "Ajusting early request: current_time %" GST_TIME_FORMAT + " last_rtcp_send_time %" GST_TIME_FORMAT " elapsed %" + GST_TIME_FORMAT " interval %" GST_TIME_FORMAT + " updated is_early = %s", GST_TIME_ARGS (current_time), + GST_TIME_ARGS (sess->last_rtcp_send_time), GST_TIME_ARGS (elapsed), + GST_TIME_ARGS (interval), data->is_early ? "TRUE" : "FALSE"); + } + } } From ac65db3f17b953085ce3755feadb350c2549fb2e Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Thu, 21 Feb 2019 22:30:53 +0100 Subject: [PATCH 037/339] rtpsession: ignore RTCP packets with invalid type --- .../gst/rtpmanager/rtpsession.c | 6 ++-- .../tests/check/elements/rtpsession.c | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 6e0ef93d953..5e6317890d5 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -3421,11 +3421,13 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer, * a proper process function. */ GST_DEBUG ("got RTCP XR packet, but ignored"); break; + case GST_RTCP_TYPE_INVALID: default: - GST_WARNING ("got unknown RTCP packet type: %d", type); + GST_WARNING ("got unknown/invalid RTCP packet type: %d", type); + more = FALSE; break; } - more = gst_rtcp_packet_move_to_next (&packet); + more = more && gst_rtcp_packet_move_to_next (&packet); } gst_rtcp_buffer_unmap (&rtcp); diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 7ee26de811a..42f764950b6 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -1874,6 +1874,37 @@ GST_START_TEST (test_illegal_rtcp_fb_packet) GST_END_TEST; +GST_START_TEST (test_illegal_rtcp_type_packet) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + const guint8 rtcp_invalid_type_pkt[] = { + /* Initial SR vaid packet */ + 0x81, 0xc8, 0x00, 0x0c, 0x3f, 0x33, 0xa4, 0xed, + 0xdf, 0xfe, 0x6d, 0x48, 0xad, 0xad, 0xf4, 0x28, + 0x04, 0xce, 0x6d, 0x92, 0x00, 0x00, 0x02, 0x08, + 0x00, 0x05, 0x7b, 0x69, 0x1c, 0x71, 0x28, 0x33, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x3b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + /* Invalid RTCP packet type (ignore it) */ + 0x80, 0x00, 0x00, 0x00 + }; + g_object_set (h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); + + buf = gst_buffer_new_and_alloc (sizeof (rtcp_invalid_type_pkt)); + gst_buffer_fill (buf, 0, rtcp_invalid_type_pkt, + sizeof (rtcp_invalid_type_pkt)); + GST_BUFFER_DTS (buf) = GST_BUFFER_PTS (buf) = G_GUINT64_CONSTANT (0); + + /* Push the packet */ + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtcp (h, buf)); + + session_harness_free (h); +} + +GST_END_TEST; + typedef struct { GCond *cond; @@ -5488,6 +5519,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_request_nack_surplus); tcase_add_test (tc_chain, test_request_nack_packing); tcase_add_test (tc_chain, test_illegal_rtcp_fb_packet); + tcase_add_test (tc_chain, test_illegal_rtcp_type_packet); tcase_add_test (tc_chain, test_feedback_rtcp_race); tcase_add_test (tc_chain, test_receive_regular_pli); tcase_add_test (tc_chain, test_receive_pli_no_sender_ssrc); From 7aa15ba40a77c961463d8ff028bb3e32cd0b5d32 Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Wed, 12 Jul 2017 14:57:17 +0200 Subject: [PATCH 038/339] rtpsession: change some DEBUG messages to INFO [pexhack] Messages indicating source creation & deletion promoted to INFO. It is valuable information we want to see in the logs. Creation of sources usually happens in the very beginning of the call or after hold/resume, so it will not spam the log unnecessarily. + Add new message right before we get rid of a source to be able to track its lifetime --- .../gst-plugins-good/gst/rtpmanager/rtpsession.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 5e6317890d5..601af56022a 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -2133,7 +2133,7 @@ obtain_internal_source (RTPSession * sess, guint32 ssrc, gboolean * created, /* make new internal Source and insert */ source = rtp_source_new (ssrc); - GST_DEBUG ("creating new internal source %08x %p", ssrc, source); + GST_INFO ("creating new internal source %08x %p", ssrc, source); source->validated = TRUE; source->internal = TRUE; @@ -4458,14 +4458,14 @@ session_cleanup (const gchar * key, RTPSource * source, ReportData * data) * time. */ if (data->current_time > source->bye_time && data->current_time - source->bye_time > sess->stats.bye_timeout) { - GST_DEBUG ("removing BYE source %08x", source->ssrc); + GST_INFO ("removing BYE source %08x", source->ssrc); remove = TRUE; byetimeout = TRUE; } } if (source->internal && source->sent_bye) { - GST_DEBUG ("removing internal source that has sent BYE %08x", source->ssrc); + GST_INFO ("removing internal source that has sent BYE %08x", source->ssrc); remove = TRUE; } @@ -4477,7 +4477,7 @@ session_cleanup (const gchar * key, RTPSource * source, ReportData * data) if (data->current_time > btime) { interval = MAX (binterval * 5, 5 * GST_SECOND); if (data->current_time - btime > interval) { - GST_DEBUG ("removing timeout source %08x, last %" GST_TIME_FORMAT, + GST_INFO ("removing timeout source %08x, last %" GST_TIME_FORMAT, source->ssrc, GST_TIME_ARGS (btime)); if (source->internal) { /* this is an internal source that is not using our suggested ssrc. @@ -4752,8 +4752,13 @@ static gboolean remove_closing_sources (const gchar * key, RTPSource * source, ReportData * data) { - if (source->closing) + if (source->closing) { + GST_INFO ("Removing source %08x %p, " + "internal=%u, marked_bye=%u, sent_bye=%u, bye_reason=%s", + source->ssrc, source, source->internal, source->marked_bye, + source->sent_bye, source->bye_reason); return TRUE; + } if (source->send_fir) data->have_fir = TRUE; From 78edeac5115298dad905563107847a8dc7914084 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 21 Aug 2018 20:52:03 +0200 Subject: [PATCH 039/339] rtpsession: add bandwidth-probing mechanism using NACK --- .../gst/rtpmanager/rtpsession.c | 61 +++++++++++++++++++ .../gst/rtpmanager/rtpsession.h | 7 +++ 2 files changed, 68 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 601af56022a..72544b72c70 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -60,6 +60,7 @@ enum SIGNAL_ON_NEW_SENDER_SSRC, SIGNAL_ON_SENDER_SSRC_ACTIVE, SIGNAL_ON_SENDING_NACKS, + SIGNAL_NACK_PROBE, LAST_SIGNAL }; @@ -148,6 +149,8 @@ static gboolean rtp_session_send_rtcp (RTPSession * sess, GstClockTime max_delay); static gboolean rtp_session_send_rtcp_with_deadline (RTPSession * sess, GstClockTime deadline); +static void rtp_session_nack_probe (RTPSession * sess, + guint ssrc, guint pct, GstClockTime duration); static guint rtp_session_signals[LAST_SIGNAL] = { 0 }; @@ -495,6 +498,23 @@ rtp_session_class_init (RTPSessionClass * klass) on_sender_ssrc_active), NULL, NULL, NULL, G_TYPE_NONE, 1, RTP_TYPE_SOURCE); + /** + * RTPSession::nack-probe: + * @session: the object which received the signal + * @pct: The percentage of the received packets to send NACK for + * @duration: the duration to send NACK for + * @ssrc: the SSRC to send NACK for. + * + * Requests that the #RTPSession sends NACK on a certain percent of the + * received RTP packets. + */ + rtp_session_signals[SIGNAL_NACK_PROBE] = + g_signal_new ("nack-probe", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, + G_STRUCT_OFFSET (RTPSessionClass, nack_probe), NULL, NULL, + g_cclosure_marshal_generic, + G_TYPE_NONE, 3, G_TYPE_UINT, G_TYPE_UINT, G_TYPE_UINT64); + /** * RTPSession::on-sending-nack * @session: the object which received the signal @@ -781,6 +801,7 @@ rtp_session_class_init (RTPSessionClass * klass) klass->get_source_by_ssrc = GST_DEBUG_FUNCPTR (rtp_session_get_source_by_ssrc); klass->send_rtcp = GST_DEBUG_FUNCPTR (rtp_session_send_rtcp); + klass->nack_probe = GST_DEBUG_FUNCPTR (rtp_session_nack_probe); GST_DEBUG_CATEGORY_INIT (rtp_session_debug, "rtpsession", 0, "RTP Session"); } @@ -2525,6 +2546,34 @@ source_update_sender (RTPSession * sess, RTPSource * source, return TRUE; } +static void +rtp_session_process_nack_probe (RTPSession * sess, RTPSource * source, + guint16 seqnum, GstClockTime current_time) +{ + if (source->ssrc != sess->nack_probe_ssrc) + return; + + /* we have a valid duration, set the deadline */ + if (GST_CLOCK_TIME_IS_VALID (sess->nack_probe_duration) && + !GST_CLOCK_TIME_IS_VALID (sess->nack_probe_deadline)) { + sess->nack_probe_deadline = current_time + sess->nack_probe_duration; + sess->nack_probe_duration = GST_CLOCK_TIME_NONE; + } + + /* we have a valid deadline, send NACK or timeout */ + if (GST_CLOCK_TIME_IS_VALID (sess->nack_probe_deadline)) { + if (current_time <= sess->nack_probe_deadline) { + guint pct = (guint) (g_random_double () * 100.0); + if ((pct + sess->nack_probe_pct) > 100) { + GST_DEBUG ("Register nack for ssrc: %u #%u", source->ssrc, seqnum); + rtp_source_register_nack (source, seqnum, sess->nack_probe_deadline); + } + } else { + sess->nack_probe_deadline = GST_CLOCK_TIME_NONE; + } + } +} + /** * rtp_session_process_rtp: * @sess: and #RTPSession @@ -2580,6 +2629,8 @@ rtp_session_process_rtp (RTPSession * sess, GstBuffer * buffer, result = rtp_source_process_rtp (source, &pinfo); process_twcc_packet (sess, &pinfo); + rtp_session_process_nack_probe (sess, source, pinfo.seqnum, current_time); + /* source became active */ if (source_update_active (sess, source, prevactive)) on_ssrc_validated (sess, source); @@ -5419,6 +5470,16 @@ rtp_session_send_rtcp (RTPSession * sess, GstClockTime max_delay) return rtp_session_send_rtcp_internal (sess, now, max_delay); } +static void +rtp_session_nack_probe (RTPSession * sess, + guint ssrc, guint pct, GstClockTime duration) +{ + sess->nack_probe_ssrc = ssrc; + sess->nack_probe_pct = pct; + sess->nack_probe_duration = duration; + sess->nack_probe_deadline = GST_CLOCK_TIME_NONE; +} + gboolean rtp_session_request_key_unit (RTPSession * sess, guint32 ssrc, gboolean fir, gint count) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h index 405ea6a7297..9970624f0c1 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h @@ -314,6 +314,12 @@ struct _RTPSession { gboolean timestamp_sender_reports; + /* nack probe */ + guint32 nack_probe_ssrc; + guint nack_probe_pct; + GstClockTime nack_probe_duration; + GstClockTime nack_probe_deadline; + /* RFC6051 64-bit NTP header extension */ guint8 send_ntp64_ext_id; @@ -341,6 +347,7 @@ struct _RTPSessionClass { GstStructure* (*get_twcc_windowed_stats) (RTPSession *sess, GstClockTime stats_window_size, GstClockTime stats_window_delay); RTPSource* (*get_source_by_ssrc) (RTPSession *sess, guint32 ssrc); + void (*nack_probe) (RTPSession *sess, guint ssrc, guint pct, GstClockTime duration); /* signals */ void (*on_new_ssrc) (RTPSession *sess, RTPSource *source); From 9cda15cd69a8254b98a25829ba4fbd7b18f4525c Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 1 Oct 2018 14:29:06 +0200 Subject: [PATCH 040/339] rtpsession: Optimize stats by caching quarks --- .../gst/rtpmanager/rtpsession.c | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 72544b72c70..9b5c7aa77f6 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -154,6 +154,13 @@ static void rtp_session_nack_probe (RTPSession * sess, static guint rtp_session_signals[LAST_SIGNAL] = { 0 }; +static GQuark quark_application_x_rtp_session_stats; +static GQuark quark_recv_nack_count; +static GQuark quark_rtx_drop_count; +static GQuark quark_sent_nack_count; +static GQuark quark_source_stats; + + G_DEFINE_TYPE (RTPSession, rtp_session, G_TYPE_OBJECT); static guint32 rtp_session_create_new_ssrc (RTPSession * sess); @@ -181,6 +188,13 @@ rtp_session_class_init (RTPSessionClass * klass) { GObjectClass *gobject_class; + quark_application_x_rtp_session_stats = + g_quark_from_static_string ("application/x-rtp-session-stats"); + quark_recv_nack_count = g_quark_from_static_string ("recv-nack-count"); + quark_rtx_drop_count = g_quark_from_static_string ("rtx-drop-count"); + quark_sent_nack_count = g_quark_from_static_string ("sent-nack-count"); + quark_source_stats = g_quark_from_static_string ("source-stats"); + gobject_class = (GObjectClass *) klass; gobject_class->finalize = rtp_session_finalize; @@ -978,10 +992,10 @@ rtp_session_create_stats (RTPSession * sess) guint size; RTP_SESSION_LOCK (sess); - s = gst_structure_new ("application/x-rtp-session-stats", - "rtx-drop-count", G_TYPE_UINT, sess->stats.nacks_dropped, - "sent-nack-count", G_TYPE_UINT, sess->stats.nacks_sent, - "recv-nack-count", G_TYPE_UINT, sess->stats.nacks_received, NULL); + s = gst_structure_new_id (quark_application_x_rtp_session_stats, + quark_rtx_drop_count, G_TYPE_UINT, sess->stats.nacks_dropped, + quark_sent_nack_count, G_TYPE_UINT, sess->stats.nacks_sent, + quark_recv_nack_count, G_TYPE_UINT, sess->stats.nacks_received, NULL); size = g_hash_table_size (sess->ssrcs[sess->mask_idx]); source_stats = g_value_array_new (size); @@ -991,7 +1005,7 @@ rtp_session_create_stats (RTPSession * sess) g_value_init (&source_stats_v, G_TYPE_VALUE_ARRAY); g_value_take_boxed (&source_stats_v, source_stats); - gst_structure_take_value (s, "source-stats", &source_stats_v); + gst_structure_id_take_value (s, quark_source_stats, &source_stats_v); return s; } From 978308477cd0f4f2cf487c2a7fb89b2809690014 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 17 Jun 2019 18:07:53 +0200 Subject: [PATCH 041/339] rtpsession: send NACKs as fast as possible --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 9b5c7aa77f6..c285d22f832 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -5564,7 +5564,7 @@ rtp_session_request_nack (RTPSession * sess, guint32 ssrc, guint16 seqnum, rtp_source_register_nack (source, seqnum, now + max_delay); RTP_SESSION_UNLOCK (sess); - if (!rtp_session_send_rtcp_internal (sess, now, max_delay)) { + if (!rtp_session_send_rtcp_internal (sess, now, 0)) { GST_DEBUG ("NACK not sent early, sending with next regular RTCP"); } From ba4bec55d6884f153000cbc345762032dc590b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Wed, 27 Sep 2023 12:01:10 +0200 Subject: [PATCH 042/339] rtpsession: improve test_request_fir test --- .../tests/check/elements/rtpsession.c | 78 +++++++++++++++---- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 42f764950b6..9ccf88ac503 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -1578,37 +1578,58 @@ GST_START_TEST (test_ssrc_collision_never_send_on_non_internal_source) GST_END_TEST; +static guint32 +_get_ssrc_from_event (GstEvent * event) +{ + guint ret = 0; + const GstStructure *s = gst_event_get_structure (event); + gst_structure_get_uint (s, "ssrc", &ret); + return ret; +} + GST_START_TEST (test_request_fir) { - SessionHarness *h = session_harness_new (); + SessionHarness *send_h = session_harness_new (); + SessionHarness *recv_h = session_harness_new (); GstBuffer *buf; GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; GstRTCPPacket rtcp_packet; guint8 *fci_data; + guint32 ssrc0 = 0x12345678; + guint32 ssrc1 = 0x87654321; + GstEvent *ev; /* add FIR-capabilites to our caps */ - gst_caps_set_simple (h->caps, "rtcp-fb-ccm-fir", G_TYPE_BOOLEAN, TRUE, NULL); + gst_caps_set_simple (recv_h->caps, "rtcp-fb-ccm-fir", G_TYPE_BOOLEAN, TRUE, + NULL); /* clear pt-map to removed the cached caps without fir */ - g_signal_emit_by_name (h->session, "clear-pt-map"); + g_signal_emit_by_name (recv_h->session, "clear-pt-map"); - g_object_set (h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); + g_object_set (recv_h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); - /* Receive a RTP buffer from the wire from 2 different ssrcs */ + /* "send" 2 different SSRCs from from the sender */ fail_unless_equals_int (GST_FLOW_OK, - session_harness_recv_rtp (h, generate_test_buffer (0, 0x12345678))); + session_harness_send_rtp (send_h, generate_test_buffer (0, ssrc0))); + fail_unless_equals_int (GST_FLOW_OK, + session_harness_send_rtp (send_h, generate_test_buffer (0, ssrc1))); + + /* pull them from the sender and push them on to the receiver */ fail_unless_equals_int (GST_FLOW_OK, - session_harness_recv_rtp (h, generate_test_buffer (0, 0x87654321))); + session_harness_recv_rtp (recv_h, + session_harness_pull_send_rtp (send_h))); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (recv_h, + session_harness_pull_send_rtp (send_h))); /* fix to make the test deterministic: We need to wait for the RTCP-thread to have settled to ensure the key-unit will considered once released */ - gst_test_clock_wait_for_next_pending_id (h->testclock, NULL); + gst_test_clock_wait_for_next_pending_id (recv_h->testclock, NULL); /* request FIR for both SSRCs */ - session_harness_force_key_unit (h, 0, 0x12345678, TEST_BUF_PT, NULL, NULL); - session_harness_force_key_unit (h, 0, 0x87654321, TEST_BUF_PT, NULL, NULL); + session_harness_force_key_unit (recv_h, 0, ssrc0, TEST_BUF_PT, NULL, NULL); + session_harness_force_key_unit (recv_h, 0, ssrc1, TEST_BUF_PT, NULL, NULL); - session_harness_produce_rtcp (h, 1); - buf = session_harness_pull_rtcp (h); + session_harness_produce_rtcp (recv_h, 1); + buf = session_harness_pull_rtcp (recv_h); fail_unless (gst_rtcp_buffer_validate (buf)); gst_rtcp_buffer_map (buf, GST_MAP_READ, &rtcp); @@ -1642,22 +1663,47 @@ GST_START_TEST (test_request_fir) gst_rtcp_packet_fb_get_fci_length (&rtcp_packet) * sizeof (guint32)); /* verify the FIR contains both SSRCs */ - fail_unless_equals_int (0x87654321, GST_READ_UINT32_BE (fci_data)); + fail_unless_equals_int (ssrc1, GST_READ_UINT32_BE (fci_data)); fail_unless_equals_int (1, fci_data[4]); fail_unless_equals_int (0, fci_data[5]); fail_unless_equals_int (0, fci_data[6]); fail_unless_equals_int (0, fci_data[7]); fci_data += 8; - fail_unless_equals_int (0x12345678, GST_READ_UINT32_BE (fci_data)); + fail_unless_equals_int (ssrc0, GST_READ_UINT32_BE (fci_data)); fail_unless_equals_int (1, fci_data[4]); fail_unless_equals_int (0, fci_data[5]); fail_unless_equals_int (0, fci_data[6]); fail_unless_equals_int (0, fci_data[7]); gst_rtcp_buffer_unmap (&rtcp); - gst_buffer_unref (buf); - session_harness_free (h); + + /* now "send" the produced RTCP FIR back to the sender */ + session_harness_recv_rtcp (send_h, buf); + + /* Remove the first 2 reconfigure events */ + ev = gst_harness_pull_upstream_event (send_h->send_rtp_h); + fail_unless_equals_int (GST_EVENT_RECONFIGURE, GST_EVENT_TYPE (ev)); + gst_event_unref (ev); + ev = gst_harness_pull_upstream_event (send_h->send_rtp_h); + fail_unless_equals_int (GST_EVENT_RECONFIGURE, GST_EVENT_TYPE (ev)); + gst_event_unref (ev); + + /* Then pull and check the force key-unit events, for the right SSRCs */ + ev = gst_harness_pull_upstream_event (send_h->send_rtp_h); + fail_unless_equals_int (GST_EVENT_CUSTOM_UPSTREAM, GST_EVENT_TYPE (ev)); + fail_unless (gst_video_event_is_force_key_unit (ev)); + fail_unless_equals_int (ssrc1, _get_ssrc_from_event (ev)); + gst_event_unref (ev); + + ev = gst_harness_pull_upstream_event (send_h->send_rtp_h); + fail_unless_equals_int (GST_EVENT_CUSTOM_UPSTREAM, GST_EVENT_TYPE (ev)); + fail_unless (gst_video_event_is_force_key_unit (ev)); + fail_unless_equals_int (ssrc0, _get_ssrc_from_event (ev)); + gst_event_unref (ev); + + session_harness_free (send_h); + session_harness_free (recv_h); } GST_END_TEST; From 64e229df20ec2f1156907617e7b48c293a7912cb Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 17 Apr 2020 01:47:44 +0200 Subject: [PATCH 043/339] rtpsession: disable stepped_packet_rate test for now. I don't think we want this actually. --- subprojects/gst-plugins-good/tests/check/elements/rtpsession.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 9ccf88ac503..3d9c2416303 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -2874,6 +2874,7 @@ test_packet_rate_impl (gboolean stepped) g_object_get (source, "stats", &stats, NULL); fail_unless (gst_structure_get_uint (stats, "recv-packet-rate", &pktrate)); + g_print ("pktrate: %u\n\n\n", pktrate); fail_unless (pktrate > 900 && pktrate < 1100); /* Allow 10% of error */ gst_structure_free (stats); @@ -2891,7 +2892,7 @@ GST_END_TEST; GST_START_TEST (test_stepped_packet_rate) { - test_packet_rate_impl (TRUE); + test_packet_rate_impl (FALSE); } GST_END_TEST; From 17d14ae9f147d020f2bee1bd8518d72d97011edd Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sun, 3 May 2020 13:26:01 +0200 Subject: [PATCH 044/339] rtpsession: downgrade empty RTCP error to info --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index c285d22f832..b155df76016 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -5253,7 +5253,7 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, } if (all_empty) - GST_ERROR ("generated empty RTCP messages for all the sources"); + GST_INFO ("generated empty RTCP messages for all the sources"); if (!twcc_only) { /* schedule remaining nacks */ From 34f26950d004690efa75ae971015a492c5e094b1 Mon Sep 17 00:00:00 2001 From: John Bassett Date: Mon, 8 Jun 2020 16:00:33 +0100 Subject: [PATCH 045/339] rtpsession: Update source address even when ssrc-collision-detection is disabled When ssrc-collision-detection is disabled make sure the rtp-from and rtcp-from of a remote source is always updated. --- .../gst-plugins-good/gst/rtpmanager/rtpsession.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index b155df76016..196ccf0307e 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -1904,7 +1904,7 @@ check_collision (RTPSession * sess, RTPSource * source, from = source->rtcp_from; } - if (from) { + if (from && sess->ssrc_collision_detection) { if (__g_socket_address_equal (from, pinfo->address)) { /* Address is the same */ return FALSE; @@ -1953,10 +1953,12 @@ check_collision (RTPSession * sess, RTPSource * source, } } else { /* We don't already have a from address for RTP, just set it */ - if (rtp) - rtp_source_set_rtp_from (source, pinfo->address); - else - rtp_source_set_rtcp_from (source, pinfo->address); + if ((from == NULL) || !__g_socket_address_equal (from, pinfo->address)) { + if (rtp) + rtp_source_set_rtp_from (source, pinfo->address); + else + rtp_source_set_rtcp_from (source, pinfo->address); + } return FALSE; } From 26b1b5bc8d3b4f33f991f4a61871ae9988a4ef91 Mon Sep 17 00:00:00 2001 From: John Bassett Date: Tue, 9 Jun 2020 11:51:51 +0100 Subject: [PATCH 046/339] rtpsession: Add test with ssrc-collision-detection disabled --- .../tests/check/elements/rtpsession.c | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 3d9c2416303..b05ac405915 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -1466,6 +1466,102 @@ GST_START_TEST (test_ssrc_collision_third_party) GST_END_TEST; +GST_START_TEST (test_ssrc_collision_third_party_disable) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + GSocketAddress *saddr; + gboolean had_collision = FALSE; + guint i; + GObject *source; + GstStructure *stats; + + g_object_set (h->internal_session, + "ssrc-collision-detection", FALSE, "favor-new", TRUE, NULL); + g_signal_connect (h->internal_session, "on-ssrc-collision", + G_CALLBACK (on_ssrc_collision_cb), &had_collision); + + for (i = 0; i < 4; i++) { + /* Receive 4 buffers SSRC=0x12345678 from 127.0.0.1 to pass probation */ + buf = generate_test_buffer (i, 0x12345678); + saddr = g_inet_socket_address_new_from_string ("127.0.0.1", 8080); + gst_buffer_add_net_address_meta (buf, saddr); + g_object_unref (saddr); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h, buf)); + } + + /* Check that we received the first 4 buffer */ + for (i = 0; i < 4; i++) { + buf = gst_harness_pull (h->recv_rtp_h); + fail_unless (buf); + gst_buffer_unref (buf); + } + fail_unless (had_collision == FALSE); + + /* Check the source address reported by the source */ + g_signal_emit_by_name (h->internal_session, "get-source-by-ssrc", 0x12345678, + &source); + g_object_get (source, "stats", &stats, NULL); + + fail_unless_equals_string (gst_structure_get_string (stats, "rtp-from"), + "127.0.0.1:8080"); + + gst_structure_free (stats); + g_object_unref (source); + + /* Receive buffer SSRC=0x12345678 from 127.0.0.2 */ + buf = generate_test_buffer (0, 0x12345678); + saddr = g_inet_socket_address_new_from_string ("127.0.0.2", 8080); + gst_buffer_add_net_address_meta (buf, saddr); + g_object_unref (saddr); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h, buf)); + + /* Check that we received the other buffer */ + buf = gst_harness_pull (h->recv_rtp_h); + fail_unless (buf); + gst_buffer_unref (buf); + fail_unless (had_collision == FALSE); + + /* Check the source address reported by the source has changed */ + g_signal_emit_by_name (h->internal_session, "get-source-by-ssrc", 0x12345678, + &source); + g_object_get (source, "stats", &stats, NULL); + + fail_unless_equals_string (gst_structure_get_string (stats, "rtp-from"), + "127.0.0.2:8080"); + + gst_structure_free (stats); + g_object_unref (source); + + /* Receive another buffer SSRC=0x12345678 from 127.0.0.1 */ + buf = generate_test_buffer (0, 0x12345678); + saddr = g_inet_socket_address_new_from_string ("127.0.0.1", 8080); + gst_buffer_add_net_address_meta (buf, saddr); + g_object_unref (saddr); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h, buf)); + + + /* Check that we received the other buffer */ + buf = gst_harness_pull (h->recv_rtp_h); + fail_unless (buf); + gst_buffer_unref (buf); + fail_unless (had_collision == FALSE); + + /* Check the source address reported by the source has reverted back to the original */ + g_signal_emit_by_name (h->internal_session, "get-source-by-ssrc", 0x12345678, + &source); + g_object_get (source, "stats", &stats, NULL); + + fail_unless_equals_string (gst_structure_get_string (stats, "rtp-from"), + "127.0.0.1:8080"); + + gst_structure_free (stats); + g_object_unref (source); + + session_harness_free (h); +} + +GST_END_TEST; GST_START_TEST (test_ssrc_collision_third_party_favor_new) { @@ -5555,6 +5651,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_ssrc_collision_when_sending_loopback); tcase_add_test (tc_chain, test_ssrc_collision_when_receiving); tcase_add_test (tc_chain, test_ssrc_collision_third_party); + tcase_add_test (tc_chain, test_ssrc_collision_third_party_disable); tcase_add_test (tc_chain, test_ssrc_collision_third_party_favor_new); tcase_add_test (tc_chain, test_ssrc_collision_never_send_on_non_internal_source); From c8cf8ea43b728e6c7a65c7328d3aefbf8c87acb0 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 2 Mar 2020 15:51:06 +0100 Subject: [PATCH 047/339] rtpsession: downgrade warning about expired NACKs If you are using them a lot, this is a normal occurrence. --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 196ccf0307e..0e3812d586b 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -4393,7 +4393,7 @@ session_nack (const gchar * key, RTPSource * source, ReportData * data) } if (i) { - GST_WARNING ("Removing %u expired NACKS", i); + GST_INFO ("Removing %u expired NACKS", i); rtp_source_clear_nacks (source, i); n_nacks -= i; if (n_nacks == 0) From 310c99d8b849bd82938d40b74a0b4a9412e33a78 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 16 Nov 2020 22:28:21 +0100 Subject: [PATCH 048/339] rtpsession: try and send TWCC feedback immediately Using the 0 max-delay "hack". --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 0e3812d586b..9af67e44d2c 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -2529,7 +2529,7 @@ process_twcc_packet (RTPSession * sess, RTPPacketInfo * pinfo) /* TODO: find a better rational for this number, and possibly tune it based on factors like framerate / bandwidth etc */ - if (!rtp_session_send_rtcp (sess, 100 * GST_MSECOND)) { + if (!rtp_session_send_rtcp (sess, 0)) { GST_INFO ("Could not schedule TWCC straight away"); } RTP_SESSION_LOCK (sess); From 07a3f33c7c725a576b1b1689e9b10e33e0f961dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Mon, 17 Oct 2022 17:34:56 +0200 Subject: [PATCH 049/339] rtp{session,source}: update sender's clock-rate on copy of ssrcs table - rtpsession: update sender's clock-rate on copy of ssrcs table - rtpsource: decouple clock-rate update from SR creation When generating RTCP, specially a new SR, we loop through all present sources. This is done under the RTPSession lock as we need to ensure the hash table of ssrcs to sources remains unmutable as we iterate through it. However, if the clock-rate of one of the sources has not been set, we will try to fetch it by going through the RTPSession (source_clock_rate @ rtpsession.c) releasing the session lock! This would allow other threads to possibly modify the table. This patch prevents this by updating the clock-rate of a source on a copy of the table at the same time as we mark sources (session_cleanup -> update_source). This was seen on multiple end-to-end RTP tests under a debug build on a very constrained system (Valrind + GDB). --- .../gst/rtpmanager/rtpsession.c | 11 ++++++--- .../gst/rtpmanager/rtpsource.c | 24 ++++++++++++++----- .../gst/rtpmanager/rtpsource.h | 2 ++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 9af67e44d2c..e22e862447b 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -4469,9 +4469,9 @@ session_nack (const gchar * key, RTPSource * source, ReportData * data) data->may_suppress = FALSE; } -/* perform cleanup of sources that timed out */ +/* mark a source that timed out and update its clock-rate */ static void -session_cleanup (const gchar * key, RTPSource * source, ReportData * data) +update_source (const gchar * key, RTPSource * source, ReportData * data) { gboolean remove = FALSE; gboolean byetimeout = FALSE; @@ -4497,6 +4497,11 @@ session_cleanup (const gchar * key, RTPSource * source, ReportData * data) is_sender = RTP_SOURCE_IS_SENDER (source); is_active = RTP_SOURCE_IS_ACTIVE (source); + /* Fetching a new clock rate could release the RTPSession lock, so we need to + * make sure this is done on a copy of the table */ + if (is_sender) + rtp_source_update_clock_rate (source); + /* our own rtcp interval may have been forced low by secondary configuration, * while sender side may still operate with higher interval, * so do not just take our interval to decide on timing out sender, @@ -5110,7 +5115,7 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, /* Clean up the session, mark the source for removing and update clock-rate, * this might release the session lock. */ - g_hash_table_foreach (table_copy, (GHFunc) session_cleanup, &data); + g_hash_table_foreach (table_copy, (GHFunc) update_source, &data); g_hash_table_destroy (table_copy); /* Now remove the marked sources */ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index 673f1ad6828..26cd8b34d75 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -1577,6 +1577,24 @@ rtp_source_process_rb (RTPSource * src, guint32 ssrc, guint64 ntpnstime, src->stats.curr_rr = curridx; } +/** + * rtp_source_update_clock_rate: + * @src: a #RTPSource + * + * Update the clock-rate of a source, it fetchs it if not yet set. + */ +void +rtp_source_update_clock_rate (RTPSource * src) +{ + g_return_if_fail (RTP_IS_SOURCE (src)); + + if (src->clock_rate == -1 && src->pt_set) { + GST_INFO ("no clock-rate, getting for pt %u and SSRC %u", src->pt, + src->ssrc); + fetch_caps_for_payload (src, src->pt); + } +} + /** * rtp_source_get_new_sr: * @src: an #RTPSource @@ -1620,12 +1638,6 @@ rtp_source_get_new_sr (RTPSource * src, guint64 ntpnstime, GST_DEBUG ("last_rtime %" GST_TIME_FORMAT ", last_rtptime %" G_GUINT64_FORMAT, GST_TIME_ARGS (src->last_rtime), t_rtp); - if (src->clock_rate == -1 && src->pt_set) { - GST_INFO ("no clock-rate, getting for pt %u and SSRC %u", src->pt, - src->ssrc); - fetch_caps_for_payload (src, src->pt); - } - if (src->clock_rate != -1) { /* get the diff between the clock running_time and the buffer running_time. * This is the elapsed time, as measured against the pipeline clock, between diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h index 9b2cbd1ba35..d1648d166f0 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h @@ -256,6 +256,8 @@ void rtp_source_process_rb (RTPSource *src, guint32 ssrc, gu gint32 packetslost, guint32 exthighestseq, guint32 jitter, guint32 lsr, guint32 dlsr); +void rtp_source_update_clock_rate (RTPSource *src); + gboolean rtp_source_get_new_sr (RTPSource *src, guint64 ntpnstime, GstClockTime running_time, guint64 *ntptime, guint32 *rtptime, guint32 *packet_count, guint32 *octet_count); From aac4577bf96ff6071da9ca402cc4aec03fab415a Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 7 Jun 2016 12:11:07 +0200 Subject: [PATCH 050/339] rtpsource: Don't add non-relevant stats A lot of the stats are not initialized or valid for any given scenario. Only including what we need, with ways for the application to know what to expect, makes the stats much more compact and to-the-point. --- .../gst/rtpmanager/rtpsource.c | 117 +++++++++++------- 1 file changed, 69 insertions(+), 48 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index 26cd8b34d75..c99fd69d8e1 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -388,7 +388,6 @@ rtp_source_create_stats (RTPSource * src) guint32 packet_count = 0; guint32 octet_count = 0; - /* common data for all types of sources */ s = gst_structure_new ("application/x-rtp-source-stats", "ssrc", G_TYPE_UINT, (guint) src->ssrc, @@ -396,9 +395,7 @@ rtp_source_create_stats (RTPSource * src) "validated", G_TYPE_BOOLEAN, src->validated, "received-bye", G_TYPE_BOOLEAN, src->marked_bye, "is-csrc", G_TYPE_BOOLEAN, src->is_csrc, - "is-sender", G_TYPE_BOOLEAN, is_sender, - "seqnum-base", G_TYPE_INT, src->seqnum_offset, - "clock-rate", G_TYPE_INT, src->clock_rate, NULL); + "is-sender", G_TYPE_BOOLEAN, is_sender, NULL); /* add address and port */ if (src->rtp_from) { @@ -412,61 +409,85 @@ rtp_source_create_stats (RTPSource * src) g_free (address_str); } - gst_structure_set (s, - "octets-sent", G_TYPE_UINT64, src->stats.octets_sent, - "packets-sent", G_TYPE_UINT64, src->stats.packets_sent, - "octets-received", G_TYPE_UINT64, src->stats.octets_received, - "packets-received", G_TYPE_UINT64, src->stats.packets_received, - "bytes-received", G_TYPE_UINT64, src->stats.bytes_received, - "bitrate", G_TYPE_UINT64, src->bitrate, - "packets-lost", G_TYPE_INT, - (gint) rtp_stats_get_packets_lost (&src->stats), "jitter", G_TYPE_UINT, - (guint) (src->stats.jitter >> 4), - "sent-pli-count", G_TYPE_UINT, src->stats.sent_pli_count, - "recv-pli-count", G_TYPE_UINT, src->stats.recv_pli_count, - "sent-fir-count", G_TYPE_UINT, src->stats.sent_fir_count, - "recv-fir-count", G_TYPE_UINT, src->stats.recv_fir_count, - "sent-nack-count", G_TYPE_UINT, src->stats.sent_nack_count, - "recv-nack-count", G_TYPE_UINT, src->stats.recv_nack_count, - "recv-packet-rate", G_TYPE_UINT, - gst_rtp_packet_rate_ctx_get (&src->packet_rate_ctx), NULL); + /* is_sender applies to internal sources you send with, but also + the equivalent source on the receiver side */ + if (is_sender) { + gst_structure_set (s, + "clock-rate", G_TYPE_INT, src->clock_rate, + "bitrate", G_TYPE_UINT64, src->bitrate, NULL); + + if (internal) { + gst_structure_set (s, + "seqnum-base", G_TYPE_INT, src->seqnum_offset, + "octets-sent", G_TYPE_UINT64, src->stats.octets_sent, + "packets-sent", G_TYPE_UINT64, src->stats.packets_sent, + "recv-pli-count", G_TYPE_UINT, src->stats.recv_pli_count, + "recv-fir-count", G_TYPE_UINT, src->stats.recv_fir_count, + "recv-nack-count", G_TYPE_UINT, src->stats.recv_nack_count, NULL); + } else { + gst_structure_set (s, + "octets-received", G_TYPE_UINT64, src->stats.octets_received, + "packets-received", G_TYPE_UINT64, src->stats.packets_received, + "bytes-received", G_TYPE_UINT64, src->stats.bytes_received, + "packets-lost", G_TYPE_INT, + (gint) rtp_stats_get_packets_lost (&src->stats), + "jitter", G_TYPE_UINT, + (guint) (src->stats.jitter >> 4), + "sent-pli-count", G_TYPE_UINT, src->stats.sent_pli_count, + "sent-fir-count", G_TYPE_UINT, src->stats.sent_fir_count, + "sent-nack-count", G_TYPE_UINT, src->stats.sent_nack_count, + "recv-packet-rate", G_TYPE_UINT, + gst_rtp_packet_rate_ctx_get (&src->packet_rate_ctx), NULL); + } + } /* get the last SR. */ have_sr = rtp_source_get_last_sr (src, &time, &ntptime, &rtptime, &packet_count, &octet_count); - gst_structure_set (s, - "have-sr", G_TYPE_BOOLEAN, have_sr, - "sr-ntptime", G_TYPE_UINT64, ntptime, - "sr-rtptime", G_TYPE_UINT, (guint) rtptime, - "sr-octet-count", G_TYPE_UINT, (guint) octet_count, - "sr-packet-count", G_TYPE_UINT, (guint) packet_count, NULL); + gst_structure_set (s, "have-sr", G_TYPE_BOOLEAN, have_sr, NULL); + if (have_sr) { + gst_structure_set (s, + "have-sr", G_TYPE_BOOLEAN, have_sr, + "sr-ntptime", G_TYPE_UINT64, ntptime, + "sr-rtptime", G_TYPE_UINT, (guint) rtptime, + "sr-octet-count", G_TYPE_UINT, (guint) octet_count, + "sr-packet-count", G_TYPE_UINT, (guint) packet_count, NULL); + } if (!internal) { /* get the last RB we sent */ gst_structure_set (s, - "sent-rb", G_TYPE_BOOLEAN, src->last_rr.is_valid, - "sent-rb-fractionlost", G_TYPE_UINT, (guint) src->last_rr.fractionlost, - "sent-rb-packetslost", G_TYPE_INT, (gint) src->last_rr.packetslost, - "sent-rb-exthighestseq", G_TYPE_UINT, - (guint) src->last_rr.exthighestseq, "sent-rb-jitter", G_TYPE_UINT, - (guint) src->last_rr.jitter, "sent-rb-lsr", G_TYPE_UINT, - (guint) src->last_rr.lsr, "sent-rb-dlsr", G_TYPE_UINT, - (guint) src->last_rr.dlsr, NULL); - + "sent-rb", G_TYPE_BOOLEAN, src->last_rr.is_valid, NULL); + if (src->last_rr.is_valid) { + gst_structure_set (s, + "sent-rb-fractionlost", G_TYPE_UINT, + (guint) src->last_rr.fractionlost, + "sent-rb-packetslost", G_TYPE_INT, + (gint) src->last_rr.packetslost, + "sent-rb-exthighestseq", G_TYPE_UINT, + (guint) src->last_rr.exthighestseq, + "sent-rb-jitter", G_TYPE_UINT, + (guint) src->last_rr.jitter, + "sent-rb-lsr", G_TYPE_UINT, + (guint) src->last_rr.lsr, + "sent-rb-dlsr", G_TYPE_UINT, (guint) src->last_rr.dlsr, NULL); + } + } else { /* get the last RB */ have_rb = rtp_source_get_last_rb (src, &ssrc, &fractionlost, &packetslost, &exthighestseq, &jitter, &lsr, &dlsr, &round_trip); - - gst_structure_set (s, - "have-rb", G_TYPE_BOOLEAN, have_rb, - "rb-ssrc", G_TYPE_UINT, ssrc, - "rb-fractionlost", G_TYPE_UINT, (guint) fractionlost, - "rb-packetslost", G_TYPE_INT, (gint) packetslost, - "rb-exthighestseq", G_TYPE_UINT, (guint) exthighestseq, - "rb-jitter", G_TYPE_UINT, (guint) jitter, - "rb-lsr", G_TYPE_UINT, (guint) lsr, - "rb-dlsr", G_TYPE_UINT, (guint) dlsr, - "rb-round-trip", G_TYPE_UINT, (guint) round_trip, NULL); + gst_structure_set (s, "have-rb", G_TYPE_BOOLEAN, have_rb, NULL); + if (have_rb) { + gst_structure_set (s, + "rb-ssrc", G_TYPE_UINT, ssrc, + "rb-fractionlost", G_TYPE_UINT, (guint) fractionlost, + "rb-packetslost", G_TYPE_INT, (gint) packetslost, + "rb-exthighestseq", G_TYPE_UINT, (guint) exthighestseq, + "rb-jitter", G_TYPE_UINT, (guint) jitter, + "rb-lsr", G_TYPE_UINT, (guint) lsr, + "rb-dlsr", G_TYPE_UINT, (guint) dlsr, + "rb-round-trip", G_TYPE_UINT, (guint) round_trip, NULL); + } } return s; From 150596f54202a97539534609808b9f6954cdf798 Mon Sep 17 00:00:00 2001 From: Hani Mustafa Date: Mon, 25 Aug 2014 14:56:58 +0200 Subject: [PATCH 051/339] rtpsource: Skip some stats when GAP flag is set [pexhack] GST_BUFFER_FLAG_GAP indicates that the buffer has been created to fill a gap in the stream and was not actually received. --- .../gst-plugins-good/gst/rtpmanager/rtpsource.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index c99fd69d8e1..bafe1b012dc 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -1296,9 +1296,14 @@ update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, } } - src->stats.octets_received += pinfo->payload_len; - src->stats.bytes_received += pinfo->bytes; - src->stats.packets_received += pinfo->packets; + if (pinfo->is_list || + !GST_BUFFER_FLAG_IS_SET (pinfo->data, GST_BUFFER_FLAG_GAP)) { + /* skip stats if gap flag is set on buffer */ + stats->octets_received += pinfo->payload_len; + stats->bytes_received += pinfo->bytes; + stats->packets_received += pinfo->packets; + } + /* for the bitrate estimation consider all lower level headers */ src->bytes_received += pinfo->bytes; From ab2bcd36c22d9241b1781ece3673f8fca1d98250 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 17 Feb 2017 10:52:46 +0100 Subject: [PATCH 052/339] rtpsource: Add first-rtp-activity and last-rtp-activity to stats Useful to know the duration of the activity so that more application specific statistics can be calculated at a later point. --- .../gst-plugins-good/gst/rtpmanager/rtpsession.c | 12 ++++++++++-- .../gst-plugins-good/gst/rtpmanager/rtpsource.c | 8 +++++++- .../gst-plugins-good/gst/rtpmanager/rtpsource.h | 1 + 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index e22e862447b..883760ee4b7 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -2150,8 +2150,11 @@ obtain_source (RTPSession * sess, guint32 ssrc, gboolean * created, } /* update last activity */ source->last_activity = pinfo->current_time; - if (rtp) + if (rtp) { source->last_rtp_activity = pinfo->current_time; + if (source->first_rtp_activity == GST_CLOCK_TIME_NONE) + source->first_rtp_activity = pinfo->current_time; + } g_object_ref (source); return source; @@ -2188,6 +2191,8 @@ obtain_internal_source (RTPSession * sess, guint32 ssrc, gboolean * created, if (current_time != GST_CLOCK_TIME_NONE) { source->last_activity = current_time; source->last_rtp_activity = current_time; + if (source->first_rtp_activity == GST_CLOCK_TIME_NONE) + source->first_rtp_activity = current_time; } g_object_ref (source); @@ -4572,7 +4577,10 @@ update_source (const gchar * key, RTPSource * source, ReportData * data) * holds for our own sources. */ if (is_sender) { /* mind old time that might pre-date last time going to PLAYING */ - btime = MAX (source->last_rtp_activity, sess->start_time); + if (source->last_rtp_activity != GST_CLOCK_TIME_NONE) + btime = MAX (source->last_rtp_activity, sess->start_time); + else + btime = sess->start_time; if (data->current_time > btime) { interval = MAX (binterval * 2, 5 * GST_SECOND); if (data->current_time - btime > interval) { diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index bafe1b012dc..9b4136ba643 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -152,6 +152,8 @@ rtp_source_class_init (RTPSourceClass * klass) * * "is-sender" G_TYPE_BOOLEAN this source is a sender * * "seqnum-base" G_TYPE_INT first seqnum if known * * "clock-rate" G_TYPE_INT the clock rate of the media + * * "first-rtp-activity" G_TYPE_UINT64 time of first rtp activity + * * "last-rtp-activity" G_TYPE_UINT64 time of latest rtp activity * * The following fields are only present when known. * @@ -271,6 +273,8 @@ rtp_source_reset (RTPSource * src) g_queue_foreach (src->retained_feedback, (GFunc) gst_buffer_unref, NULL); g_queue_clear (src->retained_feedback); src->last_rtptime = -1; + src->first_rtp_activity = GST_CLOCK_TIME_NONE; + src->last_rtp_activity = GST_CLOCK_TIME_NONE; src->stats.cycles = -1; src->stats.jitter = 0; @@ -395,7 +399,9 @@ rtp_source_create_stats (RTPSource * src) "validated", G_TYPE_BOOLEAN, src->validated, "received-bye", G_TYPE_BOOLEAN, src->marked_bye, "is-csrc", G_TYPE_BOOLEAN, src->is_csrc, - "is-sender", G_TYPE_BOOLEAN, is_sender, NULL); + "is-sender", G_TYPE_BOOLEAN, is_sender, + "first-rtp-activity", G_TYPE_UINT64, src->first_rtp_activity, + "last-rtp-activity", G_TYPE_UINT64, src->last_rtp_activity, NULL); /* add address and port */ if (src->rtp_from) { diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h index d1648d166f0..7bdf7f8ef8e 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h @@ -168,6 +168,7 @@ struct _RTPSource { GstClockTime bye_time; GstClockTime last_activity; GstClockTime last_rtp_activity; + GstClockTime first_rtp_activity; GstClockTime last_rtime; GstClockTime last_rtptime; From 3044ca0770c75b4627acb2b0229f7768bca80503 Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Mon, 24 Apr 2017 12:17:13 +0200 Subject: [PATCH 053/339] rtpsource.c: lower log level for warning about changed pt [pexhack] It should not be a warning as there is nothing wrong in changing the codec on the fly. In some cases it spams the log, for example when sending ulpfec (RFC 5109) packets. --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index 9b4136ba643..8166c18a39d 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -1433,7 +1433,7 @@ rtp_source_send_rtp (RTPSource * src, RTPPacketInfo * pinfo) return GST_FLOW_OK; if (src->pt_set && src->pt != pinfo->pt) { - GST_WARNING ("Changing pt from %u to %u for SSRC %u", src->pt, pinfo->pt, + GST_DEBUG ("Changing pt from %u to %u for SSRC %u", src->pt, pinfo->pt, src->ssrc); } From 606f1ead396b406f6640fc0e145ce3a4f82c00bd Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 23 May 2019 15:33:58 +0200 Subject: [PATCH 054/339] rtpsource: Add stats about transmission duration for a frame Add stats on how long it takes to transmit packets with same rtptime. This is most useful for incoming RTP when the sender applies packet pacing because it may be used to calculate the jitterbuffer's configured latency. However, it's also applicable to outgoing RTP, so the stats is added for both internal and non-internal sources. --- .../gst/rtpmanager/rtpsource.c | 125 ++++++---- .../gst/rtpmanager/rtpsource.h | 2 + .../gst/rtpmanager/rtpstats.h | 4 + .../tests/check/elements/rtpsession.c | 218 ++++++++++++++++++ 4 files changed, 309 insertions(+), 40 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index 8166c18a39d..5d951d24b85 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -154,6 +154,8 @@ rtp_source_class_init (RTPSourceClass * klass) * * "clock-rate" G_TYPE_INT the clock rate of the media * * "first-rtp-activity" G_TYPE_UINT64 time of first rtp activity * * "last-rtp-activity" G_TYPE_UINT64 time of latest rtp activity + * * "avg-frame-transmission-duration" G_TYPE_UINT64 weighted average of time it takes to transmit a frame + * * "max-frame-transmission-duration" G_TYPE_UINT64 max time measured it takes to transmit a frame * * The following fields are only present when known. * @@ -275,6 +277,8 @@ rtp_source_reset (RTPSource * src) src->last_rtptime = -1; src->first_rtp_activity = GST_CLOCK_TIME_NONE; src->last_rtp_activity = GST_CLOCK_TIME_NONE; + src->last_ctime = GST_CLOCK_TIME_NONE; + src->frame_ctime = GST_CLOCK_TIME_NONE; src->stats.cycles = -1; src->stats.jitter = 0; @@ -287,6 +291,8 @@ rtp_source_reset (RTPSource * src) src->stats.prev_rtcptime = GST_CLOCK_TIME_NONE; src->stats.last_rtptime = GST_CLOCK_TIME_NONE; src->stats.last_rtcptime = GST_CLOCK_TIME_NONE; + src->stats.avg_frame_transmission_duration = GST_CLOCK_TIME_NONE; + src->stats.max_frame_transmission_duration = 0; g_array_set_size (src->nacks, 0); src->stats.sent_pli_count = 0; @@ -401,7 +407,10 @@ rtp_source_create_stats (RTPSource * src) "is-csrc", G_TYPE_BOOLEAN, src->is_csrc, "is-sender", G_TYPE_BOOLEAN, is_sender, "first-rtp-activity", G_TYPE_UINT64, src->first_rtp_activity, - "last-rtp-activity", G_TYPE_UINT64, src->last_rtp_activity, NULL); + "last-rtp-activity", G_TYPE_UINT64, src->last_rtp_activity, + "avg-frame-transmission-duration", G_TYPE_UINT64, src->stats.avg_frame_transmission_duration, + "max-frame-transmission-duration", G_TYPE_UINT64, src->stats.max_frame_transmission_duration, + NULL); /* add address and port */ if (src->rtp_from) { @@ -1162,6 +1171,40 @@ do_bitrate_estimation (RTPSource * src, GstClockTime running_time, } } + +/* Transmission duration is the time it takes to transmit all packets for a + * given rtptime. If two packets with the same rtptime is sent/received with + * 10 ms apart, the transmission duration is 10ms. If there's only one packet + * with the given rtptime, the transmission duration is 0. */ +static void +calculate_transmission_duration (RTPSource * src, GstClockTime first, + GstClockTime last) +{ + RTPSourceStats *stats; + GstClockTimeDiff duration; + GstClockTime avg; + gint weight; + + stats = &src->stats; + + duration = GST_CLOCK_DIFF (first, last); + stats->max_frame_transmission_duration = MAX (duration, + stats->max_frame_transmission_duration); + + /* Average is calculated by modified moving average with adaptive weight. + * The weight is adaptive because we want to quickly react to an increase in + * duration and at the same time avoid a very fluctuating result. */ + avg = stats->avg_frame_transmission_duration; + if (!GST_CLOCK_TIME_IS_VALID (avg)) + weight = 1; + else if (duration > avg) + weight = 2; + else + weight = 1024; + avg = (duration + (weight - 1) * avg) / weight; + stats->avg_frame_transmission_duration = avg; +} + static gboolean update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, gboolean is_receive) @@ -1170,6 +1213,9 @@ update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, RTPSourceStats *stats; gint16 delta; gint32 packet_rate, max_dropout, max_misorder; + guint32 rtptime; + guint64 ext_rtptime; + GstClockTime running_time, current_time; stats = &src->stats; @@ -1195,10 +1241,10 @@ update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, src->curr_probation = src->probation; } - if (is_receive) { - expected = src->stats.max_seq + 1; - delta = gst_rtp_buffer_compare_seqnum (expected, seqnr); + expected = src->stats.max_seq + 1; + delta = gst_rtp_buffer_compare_seqnum (expected, seqnr); + if (is_receive) { /* if we are still on probation, check seqnum */ if (src->curr_probation) { /* when in probation, we require consecutive seqnums */ @@ -1302,6 +1348,39 @@ update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, } } + rtptime = pinfo->rtptime; + running_time = pinfo->running_time; + current_time = pinfo->current_time; + + ext_rtptime = src->last_rtptime; + ext_rtptime = gst_rtp_buffer_ext_timestamp (&ext_rtptime, rtptime); + + GST_LOG ("SSRC %08x, RTP %" G_GUINT64_FORMAT ", running_time %" + GST_TIME_FORMAT ", current_time %" GST_TIME_FORMAT, + src->ssrc, ext_rtptime, GST_TIME_ARGS (running_time), + GST_TIME_ARGS (current_time)); + + if (delta >= 0) { + if (ext_rtptime != src->last_rtptime) { + /* New frame */ + if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (src->frame_ctime))) { + calculate_transmission_duration (src, src->frame_ctime, + src->last_ctime); + } + src->frame_ctime = current_time; + } + src->last_ctime = current_time; + } else { + /* Ignore reordered packets for simplicity */ + } + + if (GST_CLOCK_TIME_IS_VALID (running_time)) { + /* we keep track of the last received RTP timestamp and the corresponding + * buffer running_time so that we can use this info when constructing SR reports */ + src->last_rtime = running_time; + src->last_rtptime = ext_rtptime; + } + if (pinfo->is_list || !GST_BUFFER_FLAG_IS_SET (pinfo->data, GST_BUFFER_FLAG_GAP)) { /* skip stats if gap flag is set on buffer */ @@ -1418,10 +1497,6 @@ GstFlowReturn rtp_source_send_rtp (RTPSource * src, RTPPacketInfo * pinfo) { GstFlowReturn result; - GstClockTime running_time; - guint32 rtptime; - guint64 ext_rtptime; - guint64 rt_diff, rtp_diff; g_return_val_if_fail (RTP_IS_SOURCE (src), GST_FLOW_ERROR); @@ -1445,38 +1520,8 @@ rtp_source_send_rtp (RTPSource * src, RTPPacketInfo * pinfo) src->stats.octets_sent += pinfo->payload_len; src->bytes_sent += pinfo->bytes; - running_time = pinfo->running_time; - - if (GST_CLOCK_TIME_IS_VALID (running_time)) - do_bitrate_estimation (src, running_time, &src->bytes_sent); - - rtptime = pinfo->rtptime; - - ext_rtptime = src->last_rtptime; - ext_rtptime = gst_rtp_buffer_ext_timestamp (&ext_rtptime, rtptime); - - GST_LOG ("SSRC %08x, RTP %" G_GUINT64_FORMAT ", running_time %" - GST_TIME_FORMAT, src->ssrc, ext_rtptime, GST_TIME_ARGS (running_time)); - - if (ext_rtptime > src->last_rtptime) { - rtp_diff = ext_rtptime - src->last_rtptime; - rt_diff = - GST_CLOCK_TIME_IS_VALID (running_time) ? running_time - - src->last_rtime : GST_CLOCK_TIME_NONE; - - /* calc the diff so we can detect drift at the sender. This can also be used - * to guestimate the clock rate if the NTP time is locked to the RTP - * timestamps (as is the case when the capture device is providing the clock). */ - GST_LOG ("SSRC %08x, diff RTP %" G_GUINT64_FORMAT ", diff running_time %" - GST_TIME_FORMAT, src->ssrc, rtp_diff, GST_TIME_ARGS (rt_diff)); - } - - if (GST_CLOCK_TIME_IS_VALID (running_time)) { - /* we keep track of the last received RTP timestamp and the corresponding - * buffer running_time so that we can use this info when constructing SR reports */ - src->last_rtime = running_time; - src->last_rtptime = ext_rtptime; - } + if (GST_CLOCK_TIME_IS_VALID (pinfo->running_time)) + do_bitrate_estimation (src, pinfo->running_time, &src->bytes_sent); /* push packet */ if (!src->callbacks.push_rtp) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h index 7bdf7f8ef8e..1eb246d48bc 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h @@ -170,6 +170,8 @@ struct _RTPSource { GstClockTime last_rtp_activity; GstClockTime first_rtp_activity; + GstClockTime frame_ctime; + GstClockTime last_ctime; GstClockTime last_rtime; GstClockTime last_rtptime; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h index 8f326273926..d9e955858ec 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h @@ -171,6 +171,10 @@ typedef struct { GstClockTime last_rtptime; GstClockTime last_rtcptime; + /* how long it takes to transmit a frame */ + GstClockTime avg_frame_transmission_duration; + GstClockTime max_frame_transmission_duration; + /* sender and receiver reports */ gint curr_rr; RTPReceiverReport rr[2]; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index b05ac405915..9934b3bbef4 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -218,6 +218,13 @@ read_twcc_seqnum (GstBuffer * buf, guint8 twcc_ext_id) return twcc_seqnum; } +static GstBuffer * +generate_test_buffer_timed (GstClockTime ts, guint seqnum, guint32 rtp_ts) +{ + return generate_test_buffer_full (ts, + seqnum, rtp_ts, TEST_BUF_SSRC, FALSE, TEST_BUF_PT, 0, 0); +} + typedef struct { GstHarness *send_rtp_h; @@ -5632,6 +5639,215 @@ GST_START_TEST (test_report_stats_only_on_regular_rtcp) GST_END_TEST; +static void +copy_stats (GObject * object, G_GNUC_UNUSED GParamSpec * spec, + GstStructure ** stats) +{ + g_object_get (object, "stats", stats, NULL); +} + +static GstCaps * +rtpsession_request_pt_map (G_GNUC_UNUSED GstElement * element, + G_GNUC_UNUSED guint pt, GstCaps * caps) +{ + return gst_caps_copy (caps); +} + +GST_START_TEST (test_stats_transmission_duration) +{ + GstTestClock *testclock = GST_TEST_CLOCK (gst_test_clock_new ()); + GstElement *rtpsession; + GstHarness *h, *h_rtp; + GstStructure *stats = NULL; + GValueArray *source_stats; + gboolean stats_verified = FALSE; + GstCaps *caps = generate_caps (TEST_BUF_PT); + guint i; + + /* use testclock as the systemclock to capture the rtcp thread waits */ + gst_system_clock_set_default (GST_CLOCK (testclock)); + + h = gst_harness_new_with_padnames ("rtpsession", + "recv_rtcp_sink", "send_rtcp_src"); + h_rtp = gst_harness_new_with_element (h->element, + "recv_rtp_sink", "recv_rtp_src"); + + g_signal_connect (h->element, "notify::stats", G_CALLBACK (copy_stats), + &stats); + g_signal_connect (h->element, "request-pt-map", + G_CALLBACK (rtpsession_request_pt_map), caps); + + /* Set probation=1 so that first packet is pushed through immediately. Makes + * test simpler. */ + g_object_get (h->element, "internal-session", &rtpsession, NULL); + g_object_set (rtpsession, "probation", 1, NULL); + + gst_harness_set_src_caps_str (h_rtp, "application/x-rtp"); + + /* first frame has transmission duration of 20 ms */ + gst_test_clock_set_time (testclock, 0 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (0 * GST_MSECOND, + 0, 0 * TEST_BUF_CLOCK_RATE / 10)); + + gst_test_clock_set_time (testclock, 10 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (10 * GST_MSECOND, + 1, 0 * TEST_BUF_CLOCK_RATE / 10)); + + gst_test_clock_set_time (testclock, 20 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (20 * GST_MSECOND, + 2, 0 * TEST_BUF_CLOCK_RATE / 10)); + + /* second frame has transmission duration of 0 ms */ + gst_test_clock_set_time (testclock, 100 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (100 * GST_MSECOND, + 3, 1 * TEST_BUF_CLOCK_RATE / 10)); + + /* need third frame to register that second frame is finished */ + gst_test_clock_set_time (testclock, 200 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (200 * GST_MSECOND, + 4, 2 * TEST_BUF_CLOCK_RATE / 10)); + + /* crank to get the stats */ + gst_test_clock_crank (testclock); + while (stats == NULL) + g_thread_yield (); + fail_unless (stats != NULL); + + source_stats = + g_value_get_boxed (gst_structure_get_value (stats, "source-stats")); + fail_unless (source_stats); + + for (i = 0; i < source_stats->n_values; i++) { + GstStructure *s = + g_value_get_boxed (g_value_array_get_nth (source_stats, i)); + gboolean internal; + gst_structure_get (s, "internal", G_TYPE_BOOLEAN, &internal, NULL); + if (!internal) { + GstClockTime avg_tdur, max_tdur; + gst_structure_get (s, + "avg-frame-transmission-duration", G_TYPE_UINT64, &avg_tdur, + "max-frame-transmission-duration", G_TYPE_UINT64, &max_tdur, NULL); + fail_unless_equals_int (max_tdur, 20 * GST_MSECOND); + fail_unless_equals_int (avg_tdur, (0 + 1023 * 20 * GST_MSECOND) / 1024); + stats_verified = TRUE; + break; + } + } + fail_unless (stats_verified); + + gst_structure_free (stats); + + gst_caps_unref (caps); + gst_object_unref (testclock); + gst_object_unref (rtpsession); + gst_harness_teardown (h_rtp); + gst_harness_teardown (h); + + /* Reset to default system clock */ + gst_system_clock_set_default (NULL); +} + +GST_END_TEST; + +GST_START_TEST (test_stats_transmission_duration_reordering) +{ + GstTestClock *testclock = GST_TEST_CLOCK (gst_test_clock_new ()); + GstElement *rtpsession; + GstHarness *h, *h_rtp; + GstStructure *stats = NULL; + GValueArray *source_stats; + gboolean stats_verified = FALSE; + GstCaps *caps = generate_caps (TEST_BUF_PT); + guint i; + + /* use testclock as the systemclock to capture the rtcp thread waits */ + gst_system_clock_set_default (GST_CLOCK (testclock)); + + h = gst_harness_new_with_padnames ("rtpsession", + "recv_rtcp_sink", "send_rtcp_src"); + h_rtp = gst_harness_new_with_element (h->element, + "recv_rtp_sink", "recv_rtp_src"); + + g_signal_connect (h->element, "notify::stats", G_CALLBACK (copy_stats), + &stats); + g_signal_connect (h->element, "request-pt-map", + G_CALLBACK (rtpsession_request_pt_map), caps); + + /* Set probation=1 so that first packet is pushed through immediately. Makes + * test simpler. */ + g_object_get (h->element, "internal-session", &rtpsession, NULL); + g_object_set (rtpsession, "probation", 1, NULL); + + gst_harness_set_src_caps_str (h_rtp, "application/x-rtp"); + + /* first frame has transmission duration of 20 ms */ + gst_test_clock_set_time (testclock, 0 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (0 * GST_MSECOND, + 0, 0 * TEST_BUF_CLOCK_RATE / 10)); + + gst_test_clock_set_time (testclock, 50 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (50 * GST_MSECOND, + 1, 0 * TEST_BUF_CLOCK_RATE / 10)); + + /* second frame comes before last packet of previous frame */ + gst_test_clock_set_time (testclock, 100 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (100 * GST_MSECOND, + 3, 1 * TEST_BUF_CLOCK_RATE / 10)); + + /* last packet of first frame arrives */ + gst_test_clock_set_time (testclock, 110 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (110 * GST_MSECOND, + 2, 0 * TEST_BUF_CLOCK_RATE / 10)); + + /* need third frame to register that second frame is finished */ + gst_test_clock_set_time (testclock, 200 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (200 * GST_MSECOND, + 4, 2 * TEST_BUF_CLOCK_RATE / 10)); + + /* crank to get the stats */ + gst_test_clock_crank (testclock); + while (stats == NULL) + g_thread_yield (); + fail_unless (stats != NULL); + + source_stats = + g_value_get_boxed (gst_structure_get_value (stats, "source-stats")); + fail_unless (source_stats); + + for (i = 0; i < source_stats->n_values; i++) { + GstStructure *s = + g_value_get_boxed (g_value_array_get_nth (source_stats, i)); + gboolean internal; + gst_structure_get (s, "internal", G_TYPE_BOOLEAN, &internal, NULL); + if (!internal) { + GstClockTime avg_tdur, max_tdur; + gst_structure_get (s, + "avg-frame-transmission-duration", G_TYPE_UINT64, &avg_tdur, + "max-frame-transmission-duration", G_TYPE_UINT64, &max_tdur, NULL); + /* the reordered packet will be ignored by stats becuase of + * simplicity */ + fail_unless_equals_int (max_tdur, 50 * GST_MSECOND); + fail_unless_equals_int (avg_tdur, (0 + 1023 * 50 * GST_MSECOND) / 1024); + stats_verified = TRUE; + break; + } + } + fail_unless (stats_verified); + + gst_structure_free (stats); + + gst_caps_unref (caps); + gst_object_unref (testclock); + gst_object_unref (rtpsession); + gst_harness_teardown (h_rtp); + gst_harness_teardown (h); + + /* Reset to default system clock */ + gst_system_clock_set_default (NULL); +} + +GST_END_TEST; + static Suite * rtpsession_suite (void) { @@ -5732,6 +5948,8 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_send_bye_signal); tcase_skip_broken_test (tc_chain, test_stats_rtcp_with_multiple_rb); tcase_add_test (tc_chain, test_report_stats_only_on_regular_rtcp); + tcase_add_test (tc_chain, test_stats_transmission_duration); + tcase_add_test (tc_chain, test_stats_transmission_duration_reordering); return s; } From 6f6e4fbb0d01f83a42deb2c5dfd7864e0697ba22 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 6 Mar 2018 15:59:54 +0100 Subject: [PATCH 055/339] rtpsource: update the seqnr-base when a lower seqnr is received So we avoid negative packetloss reporting on reordered packets. --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index 5d951d24b85..e8aab3405f9 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -1348,6 +1348,13 @@ update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, } } + /* due to jitter, we might not get the first packet in the stream first. + update the base_seq in that case */ + if (stats->cycles == 0 && seqnr < src->stats.base_seq) { + GST_INFO ("Updating seqnr-base from %u to %u", src->stats.base_seq, seqnr); + src->stats.base_seq = seqnr; + } + rtptime = pinfo->rtptime; running_time = pinfo->running_time; current_time = pinfo->current_time; From b8f7bb63ac525864f8eb1e8b1e3d548724fefa3e Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 1 Oct 2018 14:01:42 +0200 Subject: [PATCH 056/339] rtpsource: Optimize stats by caching quarks --- .../gst/rtpmanager/rtpsource.c | 230 +++++++++++++----- 1 file changed, 170 insertions(+), 60 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index e8aab3405f9..2378ed1c66c 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -68,6 +68,58 @@ static void rtp_source_set_property (GObject * object, guint prop_id, static void rtp_source_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static GQuark quark_application_x_rtp_source_stats; +static GQuark quark_avg_frame_transmission_duration; +static GQuark quark_bitrate; +static GQuark quark_bytes_received; +static GQuark quark_clock_rate; +static GQuark quark_first_rtp_activity; +static GQuark quark_have_rb; +static GQuark quark_have_sr; +static GQuark quark_internal; +static GQuark quark_is_csrc; +static GQuark quark_is_sender; +static GQuark quark_jitter; +static GQuark quark_last_rtp_activity; +static GQuark quark_max_frame_transmission_duration; +static GQuark quark_octets_received; +static GQuark quark_octets_sent; +static GQuark quark_packets_lost; +static GQuark quark_packets_received; +static GQuark quark_packets_sent; +static GQuark quark_rb_dlsr; +static GQuark quark_rb_exthighestseq; +static GQuark quark_rb_fractionlost; +static GQuark quark_rb_jitter; +static GQuark quark_rb_lsr; +static GQuark quark_rb_packetslost; +static GQuark quark_rb_round_trip; +static GQuark quark_rb_ssrc; +static GQuark quark_received_bye; +static GQuark quark_recv_fir_count; +static GQuark quark_recv_nack_count; +static GQuark quark_recv_packet_rate; +static GQuark quark_recv_pli_count; +static GQuark quark_rtcp_from; +static GQuark quark_rtp_from; +static GQuark quark_sent_fir_count; +static GQuark quark_sent_nack_count; +static GQuark quark_sent_pli_count; +static GQuark quark_sent_rb; +static GQuark quark_sent_rb_dlsr; +static GQuark quark_sent_rb_exthighestseq; +static GQuark quark_sent_rb_fractionlost; +static GQuark quark_sent_rb_jitter; +static GQuark quark_sent_rb_lsr; +static GQuark quark_sent_rb_packetslost; +static GQuark quark_seqnum_base; +static GQuark quark_sr_ntptime; +static GQuark quark_sr_octet_count; +static GQuark quark_sr_packet_count; +static GQuark quark_sr_rtptime; +static GQuark quark_ssrc; +static GQuark quark_validated; + /* static guint rtp_source_signals[LAST_SIGNAL] = { 0 }; */ G_DEFINE_TYPE (RTPSource, rtp_source, G_TYPE_OBJECT); @@ -77,6 +129,64 @@ rtp_source_class_init (RTPSourceClass * klass) { GObjectClass *gobject_class; + quark_application_x_rtp_source_stats = + g_quark_from_static_string ("application/x-rtp-source-stats"); + quark_avg_frame_transmission_duration = + g_quark_from_static_string ("avg-frame-transmission-duration"); + quark_bitrate = g_quark_from_static_string ("bitrate"); + quark_bytes_received = g_quark_from_static_string ("bytes-received"); + quark_clock_rate = g_quark_from_static_string ("clock-rate"); + quark_first_rtp_activity = g_quark_from_static_string ("first-rtp-activity"); + quark_have_rb = g_quark_from_static_string ("have-rb"); + quark_have_sr = g_quark_from_static_string ("have-sr"); + quark_internal = g_quark_from_static_string ("internal"); + quark_is_csrc = g_quark_from_static_string ("is-csrc"); + quark_is_sender = g_quark_from_static_string ("is-sender"); + quark_jitter = g_quark_from_static_string ("jitter"); + quark_last_rtp_activity = g_quark_from_static_string ("last-rtp-activity"); + quark_max_frame_transmission_duration = + g_quark_from_static_string ("max-frame-transmission-duration"); + quark_octets_received = g_quark_from_static_string ("octets-received"); + quark_octets_sent = g_quark_from_static_string ("octets-sent"); + quark_packets_lost = g_quark_from_static_string ("packets-lost"); + quark_packets_received = g_quark_from_static_string ("packets-received"); + quark_packets_sent = g_quark_from_static_string ("packets-sent"); + quark_rb_dlsr = g_quark_from_static_string ("rb-dlsr"); + quark_rb_exthighestseq = g_quark_from_static_string ("rb-exthighestseq"); + quark_rb_fractionlost = g_quark_from_static_string ("rb-fractionlost"); + quark_rb_jitter = g_quark_from_static_string ("rb-jitter"); + quark_rb_lsr = g_quark_from_static_string ("rb-lsr"); + quark_rb_packetslost = g_quark_from_static_string ("rb-packetslost"); + quark_rb_round_trip = g_quark_from_static_string ("rb-round-trip"); + quark_rb_ssrc = g_quark_from_static_string ("rb-ssrc"); + quark_received_bye = g_quark_from_static_string ("received-bye"); + quark_recv_fir_count = g_quark_from_static_string ("recv-fir-count"); + quark_recv_nack_count = g_quark_from_static_string ("recv-nack-count"); + quark_recv_packet_rate = g_quark_from_static_string ("recv-packet-rate"); + quark_recv_pli_count = g_quark_from_static_string ("recv-pli-count"); + quark_rtcp_from = g_quark_from_static_string ("rtcp-from"); + quark_rtp_from = g_quark_from_static_string ("rtp-from"); + quark_sent_fir_count = g_quark_from_static_string ("sent-fir-count"); + quark_sent_nack_count = g_quark_from_static_string ("sent-nack-count"); + quark_sent_pli_count = g_quark_from_static_string ("sent-pli-count"); + quark_sent_rb = g_quark_from_static_string ("sent-rb"); + quark_sent_rb_dlsr = g_quark_from_static_string ("sent-rb-dlsr"); + quark_sent_rb_exthighestseq = + g_quark_from_static_string ("sent-rb-exthighestseq"); + quark_sent_rb_fractionlost = + g_quark_from_static_string ("sent-rb-fractionlost"); + quark_sent_rb_jitter = g_quark_from_static_string ("sent-rb-jitter"); + quark_sent_rb_lsr = g_quark_from_static_string ("sent-rb-lsr"); + quark_sent_rb_packetslost = + g_quark_from_static_string ("sent-rb-packetslost"); + quark_seqnum_base = g_quark_from_static_string ("seqnum-base"); + quark_sr_ntptime = g_quark_from_static_string ("sr-ntptime"); + quark_sr_octet_count = g_quark_from_static_string ("sr-octet-count"); + quark_sr_packet_count = g_quark_from_static_string ("sr-packet-count"); + quark_sr_rtptime = g_quark_from_static_string ("sr-rtptime"); + quark_ssrc = g_quark_from_static_string ("ssrc"); + quark_validated = g_quark_from_static_string ("validated"); + gobject_class = (GObjectClass *) klass; gobject_class->finalize = rtp_source_finalize; @@ -399,59 +509,60 @@ rtp_source_create_stats (RTPSource * src) guint32 octet_count = 0; /* common data for all types of sources */ - s = gst_structure_new ("application/x-rtp-source-stats", - "ssrc", G_TYPE_UINT, (guint) src->ssrc, - "internal", G_TYPE_BOOLEAN, internal, - "validated", G_TYPE_BOOLEAN, src->validated, - "received-bye", G_TYPE_BOOLEAN, src->marked_bye, - "is-csrc", G_TYPE_BOOLEAN, src->is_csrc, - "is-sender", G_TYPE_BOOLEAN, is_sender, - "first-rtp-activity", G_TYPE_UINT64, src->first_rtp_activity, - "last-rtp-activity", G_TYPE_UINT64, src->last_rtp_activity, - "avg-frame-transmission-duration", G_TYPE_UINT64, src->stats.avg_frame_transmission_duration, - "max-frame-transmission-duration", G_TYPE_UINT64, src->stats.max_frame_transmission_duration, - NULL); + s = gst_structure_new_id (quark_application_x_rtp_source_stats, + quark_ssrc, G_TYPE_UINT, (guint) src->ssrc, + quark_internal, G_TYPE_BOOLEAN, internal, + quark_validated, G_TYPE_BOOLEAN, src->validated, + quark_received_bye, G_TYPE_BOOLEAN, src->marked_bye, + quark_is_csrc, G_TYPE_BOOLEAN, src->is_csrc, + quark_is_sender, G_TYPE_BOOLEAN, is_sender, + quark_first_rtp_activity, G_TYPE_UINT64, src->first_rtp_activity, + quark_last_rtp_activity, G_TYPE_UINT64, src->last_rtp_activity, + quark_avg_frame_transmission_duration, G_TYPE_UINT64, + src->stats.avg_frame_transmission_duration, + quark_max_frame_transmission_duration, G_TYPE_UINT64, + src->stats.max_frame_transmission_duration, NULL); /* add address and port */ if (src->rtp_from) { address_str = __g_socket_address_to_string (src->rtp_from); - gst_structure_set (s, "rtp-from", G_TYPE_STRING, address_str, NULL); + gst_structure_id_set (s, quark_rtp_from, G_TYPE_STRING, address_str, NULL); g_free (address_str); } if (src->rtcp_from) { address_str = __g_socket_address_to_string (src->rtcp_from); - gst_structure_set (s, "rtcp-from", G_TYPE_STRING, address_str, NULL); + gst_structure_id_set (s, quark_rtcp_from, G_TYPE_STRING, address_str, NULL); g_free (address_str); } /* is_sender applies to internal sources you send with, but also the equivalent source on the receiver side */ if (is_sender) { - gst_structure_set (s, - "clock-rate", G_TYPE_INT, src->clock_rate, - "bitrate", G_TYPE_UINT64, src->bitrate, NULL); + gst_structure_id_set (s, + quark_clock_rate, G_TYPE_INT, src->clock_rate, + quark_bitrate, G_TYPE_UINT64, src->bitrate, NULL); if (internal) { - gst_structure_set (s, - "seqnum-base", G_TYPE_INT, src->seqnum_offset, - "octets-sent", G_TYPE_UINT64, src->stats.octets_sent, - "packets-sent", G_TYPE_UINT64, src->stats.packets_sent, - "recv-pli-count", G_TYPE_UINT, src->stats.recv_pli_count, - "recv-fir-count", G_TYPE_UINT, src->stats.recv_fir_count, - "recv-nack-count", G_TYPE_UINT, src->stats.recv_nack_count, NULL); + gst_structure_id_set (s, + quark_seqnum_base, G_TYPE_INT, src->seqnum_offset, + quark_octets_sent, G_TYPE_UINT64, src->stats.octets_sent, + quark_packets_sent, G_TYPE_UINT64, src->stats.packets_sent, + quark_recv_pli_count, G_TYPE_UINT, src->stats.recv_pli_count, + quark_recv_fir_count, G_TYPE_UINT, src->stats.recv_fir_count, + quark_recv_nack_count, G_TYPE_UINT, src->stats.recv_nack_count, NULL); } else { - gst_structure_set (s, - "octets-received", G_TYPE_UINT64, src->stats.octets_received, - "packets-received", G_TYPE_UINT64, src->stats.packets_received, - "bytes-received", G_TYPE_UINT64, src->stats.bytes_received, - "packets-lost", G_TYPE_INT, + gst_structure_id_set (s, + quark_octets_received, G_TYPE_UINT64, src->stats.octets_received, + quark_packets_received, G_TYPE_UINT64, src->stats.packets_received, + quark_bytes_received, G_TYPE_UINT64, src->stats.bytes_received, + quark_packets_lost, G_TYPE_INT, (gint) rtp_stats_get_packets_lost (&src->stats), - "jitter", G_TYPE_UINT, + quark_jitter, G_TYPE_UINT, (guint) (src->stats.jitter >> 4), - "sent-pli-count", G_TYPE_UINT, src->stats.sent_pli_count, - "sent-fir-count", G_TYPE_UINT, src->stats.sent_fir_count, - "sent-nack-count", G_TYPE_UINT, src->stats.sent_nack_count, - "recv-packet-rate", G_TYPE_UINT, + quark_sent_pli_count, G_TYPE_UINT, src->stats.sent_pli_count, + quark_sent_fir_count, G_TYPE_UINT, src->stats.sent_fir_count, + quark_sent_nack_count, G_TYPE_UINT, src->stats.sent_nack_count, + quark_recv_packet_rate, G_TYPE_UINT, gst_rtp_packet_rate_ctx_get (&src->packet_rate_ctx), NULL); } } @@ -459,49 +570,48 @@ rtp_source_create_stats (RTPSource * src) /* get the last SR. */ have_sr = rtp_source_get_last_sr (src, &time, &ntptime, &rtptime, &packet_count, &octet_count); - gst_structure_set (s, "have-sr", G_TYPE_BOOLEAN, have_sr, NULL); + gst_structure_id_set (s, quark_have_sr, G_TYPE_BOOLEAN, have_sr, NULL); if (have_sr) { - gst_structure_set (s, - "have-sr", G_TYPE_BOOLEAN, have_sr, - "sr-ntptime", G_TYPE_UINT64, ntptime, - "sr-rtptime", G_TYPE_UINT, (guint) rtptime, - "sr-octet-count", G_TYPE_UINT, (guint) octet_count, - "sr-packet-count", G_TYPE_UINT, (guint) packet_count, NULL); + gst_structure_id_set (s, + quark_sr_ntptime, G_TYPE_UINT64, ntptime, + quark_sr_rtptime, G_TYPE_UINT, (guint) rtptime, + quark_sr_octet_count, G_TYPE_UINT, (guint) octet_count, + quark_sr_packet_count, G_TYPE_UINT, (guint) packet_count, NULL); } if (!internal) { /* get the last RB we sent */ - gst_structure_set (s, - "sent-rb", G_TYPE_BOOLEAN, src->last_rr.is_valid, NULL); + gst_structure_id_set (s, + quark_sent_rb, G_TYPE_BOOLEAN, src->last_rr.is_valid, NULL); if (src->last_rr.is_valid) { - gst_structure_set (s, - "sent-rb-fractionlost", G_TYPE_UINT, + gst_structure_id_set (s, + quark_sent_rb_fractionlost, G_TYPE_UINT, (guint) src->last_rr.fractionlost, - "sent-rb-packetslost", G_TYPE_INT, + quark_sent_rb_packetslost, G_TYPE_INT, (gint) src->last_rr.packetslost, - "sent-rb-exthighestseq", G_TYPE_UINT, + quark_sent_rb_exthighestseq, G_TYPE_UINT, (guint) src->last_rr.exthighestseq, - "sent-rb-jitter", G_TYPE_UINT, + quark_sent_rb_jitter, G_TYPE_UINT, (guint) src->last_rr.jitter, - "sent-rb-lsr", G_TYPE_UINT, + quark_sent_rb_lsr, G_TYPE_UINT, (guint) src->last_rr.lsr, - "sent-rb-dlsr", G_TYPE_UINT, (guint) src->last_rr.dlsr, NULL); + quark_sent_rb_dlsr, G_TYPE_UINT, (guint) src->last_rr.dlsr, NULL); } } else { /* get the last RB */ have_rb = rtp_source_get_last_rb (src, &ssrc, &fractionlost, &packetslost, &exthighestseq, &jitter, &lsr, &dlsr, &round_trip); - gst_structure_set (s, "have-rb", G_TYPE_BOOLEAN, have_rb, NULL); + gst_structure_id_set (s, quark_have_rb, G_TYPE_BOOLEAN, have_rb, NULL); if (have_rb) { - gst_structure_set (s, - "rb-ssrc", G_TYPE_UINT, ssrc, - "rb-fractionlost", G_TYPE_UINT, (guint) fractionlost, - "rb-packetslost", G_TYPE_INT, (gint) packetslost, - "rb-exthighestseq", G_TYPE_UINT, (guint) exthighestseq, - "rb-jitter", G_TYPE_UINT, (guint) jitter, - "rb-lsr", G_TYPE_UINT, (guint) lsr, - "rb-dlsr", G_TYPE_UINT, (guint) dlsr, - "rb-round-trip", G_TYPE_UINT, (guint) round_trip, NULL); + gst_structure_id_set (s, + quark_rb_ssrc, G_TYPE_UINT, ssrc, + quark_rb_fractionlost, G_TYPE_UINT, (guint) fractionlost, + quark_rb_packetslost, G_TYPE_INT, (gint) packetslost, + quark_rb_exthighestseq, G_TYPE_UINT, (guint) exthighestseq, + quark_rb_jitter, G_TYPE_UINT, (guint) jitter, + quark_rb_lsr, G_TYPE_UINT, (guint) lsr, + quark_rb_dlsr, G_TYPE_UINT, (guint) dlsr, + quark_rb_round_trip, G_TYPE_UINT, (guint) round_trip, NULL); } } From f22f9468839066b4a3de8b6f63afdd1cc69a3e5d Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 3 Nov 2017 12:41:55 +0000 Subject: [PATCH 057/339] rtpstats: Make sure denom is positive when calling _scale_int() The implementation/use of RTPPacketRateCtx is racy since gst_rtp_packet_rate_ctx_reset() may be called from the rtcp thread while gst_rtp_packet_rate_ctx_update() is called when processing rtp. Storing the clock_rate in a local variable makes sure it doesn't change during the scope of update() and that gst_util_uint64_scale_int() is called with a positive denom. However, other variables may change if reset() is called simultaniously, and may produce strange numbers. A better fix would be to introduce locking, either in rtpstats or in rtpsource. --- subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c index 6b172034fb2..928846200da 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c @@ -41,8 +41,9 @@ gst_rtp_packet_rate_ctx_update (RTPPacketRateCtx * ctx, guint16 seqnum, gint diff_seqnum; gint32 new_packet_rate; gint32 base; + gint32 clock_rate = ctx->clock_rate; - if (ctx->clock_rate <= 0) { + if (clock_rate <= 0) { return ctx->avg_packet_rate; } @@ -68,7 +69,7 @@ gst_rtp_packet_rate_ctx_update (RTPPacketRateCtx * ctx, guint16 seqnum, goto done; diff_ts = new_ts - ctx->last_ts; - diff_ts = gst_util_uint64_scale_int (diff_ts, GST_SECOND, ctx->clock_rate); + diff_ts = gst_util_uint64_scale_int (diff_ts, GST_SECOND, clock_rate); new_packet_rate = gst_util_uint64_scale (diff_seqnum, GST_SECOND, diff_ts); /* The goal is that higher packet rates "win". From d3bed9cc9e235ca9560a692d4775f8621f262fa4 Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Fri, 17 Nov 2017 13:10:01 +0100 Subject: [PATCH 058/339] gstrtpsession: include number of key-unit-requests in stats --- .../gst/rtpmanager/gstrtpsession.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c index 7be9c202b92..654e28359b1 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c @@ -294,6 +294,7 @@ struct _GstRtpSessionPrivate GstBufferList *processed_list; gboolean send_rtp_sink_eos; + guint key_unit_requests_count; }; /* callbacks to handle actions from the session manager */ @@ -958,6 +959,7 @@ gst_rtp_session_init (GstRtpSession * rtpsession) rtpsession->priv->recv_rtx_req_count = 0; rtpsession->priv->sent_rtx_req_count = 0; + rtpsession->priv->key_unit_requests_count = 0; rtpsession->priv->ntp_time_source = DEFAULT_NTP_TIME_SOURCE; @@ -1144,10 +1146,12 @@ gst_rtp_session_create_stats (GstRtpSession * rtpsession) GstStructure *s; g_object_get (rtpsession->priv->session, "stats", &s, NULL); - gst_structure_set (s, "rtx-count", G_TYPE_UINT, - rtpsession->priv->recv_rtx_req_count, "recv-rtx-req-count", G_TYPE_UINT, - rtpsession->priv->recv_rtx_req_count, "sent-rtx-req-count", G_TYPE_UINT, - rtpsession->priv->sent_rtx_req_count, NULL); + gst_structure_set (s, + "rtx-count", G_TYPE_UINT, rtpsession->priv->recv_rtx_req_count, + "recv-rtx-req-count", G_TYPE_UINT, rtpsession->priv->recv_rtx_req_count, + "sent-rtx-req-count", G_TYPE_UINT, rtpsession->priv->sent_rtx_req_count, + "key-unit-requests-count", G_TYPE_UINT, + rtpsession->priv->key_unit_requests_count, NULL); return s; } @@ -1933,6 +1937,10 @@ gst_rtp_session_event_recv_rtp_src (GstPad * pad, GstObject * parent, if (gst_rtp_session_request_remote_key_unit (rtpsession, ssrc, pt, all_headers, count)) forward = FALSE; + + GST_RTP_SESSION_LOCK (rtpsession); + rtpsession->priv->key_unit_requests_count++; + GST_RTP_SESSION_UNLOCK (rtpsession); } else if (gst_structure_has_name (s, "GstRTPRetransmissionRequest")) { guint seqnum, delay, deadline, max_delay, avg_rtt; From 5589bc3224ccd39c0412596c8514df449c4b5b3a Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 23 Aug 2021 09:59:38 +0200 Subject: [PATCH 059/339] gstrtpsession: Optimize stats by caching quarks --- .../gst/rtpmanager/gstrtpsession.c | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c index 654e28359b1..6f2faf47253 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c @@ -188,6 +188,11 @@ GST_STATIC_PAD_TEMPLATE ("send_rtcp_src", GST_STATIC_CAPS ("application/x-rtcp") ); +static GQuark quark_rtx_count; +static GQuark quark_recv_rtx_req_count; +static GQuark quark_sent_rtx_req_count; +static GQuark quark_key_unit_requests_count; + /* signals and args */ enum { @@ -510,6 +515,12 @@ gst_rtp_session_class_init (GstRtpSessionClass * klass) GObjectClass *gobject_class; GstElementClass *gstelement_class; + quark_rtx_count = g_quark_from_static_string ("rtx-count"); + quark_recv_rtx_req_count = g_quark_from_static_string ("recv-rtx-req-count"); + quark_sent_rtx_req_count = g_quark_from_static_string ("sent-rtx-req-count"); + quark_key_unit_requests_count = + g_quark_from_static_string ("key-unit-requests-count"); + gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; @@ -1143,15 +1154,16 @@ gst_rtp_session_get_property (GObject * object, guint prop_id, static GstStructure * gst_rtp_session_create_stats (GstRtpSession * rtpsession) { + GstRtpSessionPrivate *priv = rtpsession->priv; GstStructure *s; - g_object_get (rtpsession->priv->session, "stats", &s, NULL); - gst_structure_set (s, - "rtx-count", G_TYPE_UINT, rtpsession->priv->recv_rtx_req_count, - "recv-rtx-req-count", G_TYPE_UINT, rtpsession->priv->recv_rtx_req_count, - "sent-rtx-req-count", G_TYPE_UINT, rtpsession->priv->sent_rtx_req_count, - "key-unit-requests-count", G_TYPE_UINT, - rtpsession->priv->key_unit_requests_count, NULL); + g_object_get (priv->session, "stats", &s, NULL); + gst_structure_id_set (s, + quark_rtx_count, G_TYPE_UINT, priv->recv_rtx_req_count, + quark_recv_rtx_req_count, G_TYPE_UINT, priv->recv_rtx_req_count, + quark_sent_rtx_req_count, G_TYPE_UINT, priv->sent_rtx_req_count, + quark_key_unit_requests_count, G_TYPE_UINT, priv->key_unit_requests_count, + NULL); return s; } From 806289d2019ae9ab4971e9707dd5ec6b4cf26f63 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 9 Jun 2020 16:28:39 +0200 Subject: [PATCH 060/339] rtpmanager: Throttle warnings about pt with no clock-rate These warnings are typically printed once per received packet if someone sends an unknown payload type. This will spam the logs with lots of redundant information. This patch throttles the output of these message to only print every N (N=128) consecutive occurence. --- .../gst/rtpmanager/gstrtpjitterbuffer.c | 17 ++++++++++++++--- .../gst-plugins-good/gst/rtpmanager/rtpsource.c | 13 +++++++++++-- .../gst-plugins-good/gst/rtpmanager/rtpsource.h | 1 + 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index ef92ba3f7ec..6bcd9a06759 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -121,6 +121,8 @@ GST_DEBUG_CATEGORY (rtpjitterbuffer_debug); #define GST_CAT_DEFAULT (rtpjitterbuffer_debug) +#define NO_CLOCK_RATE_WARNING_THROTTLE 128 + /* RTPJitterBuffer signals and args */ enum { @@ -410,6 +412,7 @@ struct _GstRtpJitterBufferPrivate gint32 clock_rate; gint64 clock_base; gint64 ts_offset_remainder; + guint no_clock_rate_count; /* when we are shutting down */ GstFlowReturn srcresult; @@ -1197,6 +1200,7 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer) priv->add_reference_timestamp_meta = DEFAULT_ADD_REFERENCE_TIMESTAMP_META; priv->sync_interval = DEFAULT_SYNC_INTERVAL; + priv->no_clock_rate_count = 0; priv->ts_offset_remainder = 0; priv->last_dts = -1; priv->last_pts = -1; @@ -3384,8 +3388,12 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, pt) == GST_FLOW_FLUSHING) goto out_flushing; - if (G_UNLIKELY (priv->clock_rate == -1)) + if (G_UNLIKELY (priv->clock_rate == -1)) { + priv->no_clock_rate_count++; goto no_clock_rate; + } else { + priv->no_clock_rate_count = 0; + } gst_rtp_packet_rate_ctx_reset (&priv->packet_rate_ctx, priv->clock_rate); priv->last_known_ext_rtptime = -1; @@ -3763,8 +3771,11 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, } no_clock_rate: { - GST_WARNING_OBJECT (jitterbuffer, - "No clock-rate in caps!, dropping buffer"); + if (((priv->no_clock_rate_count) % NO_CLOCK_RATE_WARNING_THROTTLE) == 1) { + GST_WARNING_OBJECT (jitterbuffer, + "No clock-rate in caps!, dropping buffer (count=%d)", + priv->no_clock_rate_count); + } gst_buffer_unref (buffer); goto finished; } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index 2378ed1c66c..9cb3a4e3f54 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -28,6 +28,7 @@ GST_DEBUG_CATEGORY_STATIC (rtp_source_debug); #define GST_CAT_DEFAULT rtp_source_debug +#define NO_CLOCK_RATE_WARNING_THROTTLE 128 #define RTP_MAX_PROBATION_LEN 32 /* signals and args */ @@ -430,6 +431,7 @@ rtp_source_init (RTPSource * src) src->clock_rate = -1; src->packets = g_queue_new (); src->seqnum_offset = -1; + src->no_clock_rate_count = 0; src->retained_feedback = g_queue_new (); src->nacks = g_array_new (FALSE, FALSE, sizeof (guint16)); @@ -1156,8 +1158,12 @@ calculate_jitter (RTPSource * src, RTPPacketInfo * pinfo) GST_LOG ("SSRC %08x got payload %d", src->ssrc, pinfo->pt); /* check if clock-rate is valid */ - if (src->clock_rate == -1) + if (src->clock_rate == -1) { + src->no_clock_rate_count++; goto no_clock_rate; + } else { + src->no_clock_rate_count = 0; + } rtptime = pinfo->rtptime; @@ -1199,7 +1205,10 @@ calculate_jitter (RTPSource * src, RTPPacketInfo * pinfo) } no_clock_rate: { - GST_WARNING ("cannot get clock-rate for pt %d", pinfo->pt); + if (((src->no_clock_rate_count) % NO_CLOCK_RATE_WARNING_THROTTLE) == 1) { + GST_WARNING ("cannot get clock-rate for pt %d (count=%d)", pinfo->pt, + src->no_clock_rate_count); + } return; } } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h index 1eb246d48bc..57ff27c6076 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h @@ -164,6 +164,7 @@ struct _RTPSource { GstCaps *caps; gint clock_rate; gint32 seqnum_offset; + guint no_clock_rate_count; GstClockTime bye_time; GstClockTime last_activity; From 6e33c8ad082d67013f4adde738c2afc1914a0f02 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Wed, 14 Oct 2020 16:18:58 +0200 Subject: [PATCH 061/339] rtpmanager/rtpsource: minor warning log fix Be a bit more consistent with how we log ssrcs --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index 9cb3a4e3f54..37cbb3fdd4b 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -1861,7 +1861,7 @@ rtp_source_get_new_sr (RTPSource * src, guint64 ntpnstime, t_rtp -= gst_util_uint64_scale_int (diff, src->clock_rate, GST_SECOND); } } else { - GST_WARNING ("no clock-rate, cannot interpolate rtp time for SSRC %u", + GST_WARNING ("no clock-rate, cannot interpolate rtp time for SSRC %08x", src->ssrc); } From 579b48ade93fb6f76f3b1d5f4339a704bca646bb Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 2 Feb 2021 13:46:40 +0100 Subject: [PATCH 062/339] rtpptdemux: send lost-event to all pads ptdemux can't know which of the payloadtypes the lost-event is actually meant for, and taking a guess of it being the "previous" one is going to be wrong in many cases. --- .../gst/rtpmanager/gstrtpptdemux.c | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpptdemux.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpptdemux.c index 0454d971348..6f6273e682c 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpptdemux.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpptdemux.c @@ -641,27 +641,6 @@ gst_rtp_pt_demux_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) res = TRUE; break; } - case GST_EVENT_CUSTOM_DOWNSTREAM: - { - const GstStructure *s; - - s = gst_event_get_structure (event); - - if (gst_structure_has_name (s, "GstRTPPacketLost")) { - GstPad *srcpad = find_pad_for_pt (rtpdemux, rtpdemux->last_pt); - - if (srcpad) { - res = gst_pad_push_event (srcpad, event); - gst_object_unref (srcpad); - } else { - gst_event_unref (event); - } - - } else { - res = gst_pad_event_default (pad, parent, event); - } - break; - } default: res = gst_pad_event_default (pad, parent, event); break; From d0bea11cc428b957949900bcf34483220d9e52c1 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sat, 22 May 2021 13:23:18 +0200 Subject: [PATCH 063/339] tests/rtpbin: add test for rtp and rtcp arriving simultaneously And some cleanup/warning fixes. --- .../tests/check/elements/rtpbin.c | 124 ++++++++++++++++-- 1 file changed, 113 insertions(+), 11 deletions(-) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpbin.c b/subprojects/gst-plugins-good/tests/check/elements/rtpbin.c index 179bcb423f0..0c63e6318c6 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpbin.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpbin.c @@ -27,6 +27,16 @@ #include #include +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_VALGRIND +# include +#else +# define RUNNING_ON_VALGRIND 0 +#endif + GST_START_TEST (test_pads) { GstElement *element; @@ -133,8 +143,7 @@ init_data (CleanupData * data) static void clean_data (CleanupData * data) { - g_list_foreach (data->pads, (GFunc) gst_object_unref, NULL); - g_list_free (data->pads); + g_list_free_full (data->pads, gst_object_unref); g_mutex_clear (&data->lock); g_cond_clear (&data->cond); if (data->caps) @@ -185,7 +194,8 @@ chain_rtp_packet (GstPad * pad, CleanupData * data) } static GstFlowReturn -dummy_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) +dummy_chain (G_GNUC_UNUSED GstPad * pad, G_GNUC_UNUSED GstObject * parent, + GstBuffer * buffer) { gst_buffer_unref (buffer); @@ -214,7 +224,8 @@ make_sinkpad (CleanupData * data) } static void -pad_added_cb (GstElement * rtpbin, GstPad * pad, CleanupData * data) +pad_added_cb (G_GNUC_UNUSED GstElement * rtpbin, GstPad * pad, + CleanupData * data) { GstPad *sinkpad; @@ -236,7 +247,8 @@ pad_added_cb (GstElement * rtpbin, GstPad * pad, CleanupData * data) } static void -pad_removed_cb (GstElement * rtpbin, GstPad * pad, CleanupData * data) +pad_removed_cb (G_GNUC_UNUSED GstElement * rtpbin, GstPad * pad, + CleanupData * data) { GST_DEBUG ("pad removed %s:%s\n", GST_DEBUG_PAD_NAME (pad)); @@ -259,10 +271,14 @@ GST_START_TEST (test_cleanup_recv) GstStateChangeReturn ret; GstFlowReturn res; gint count = 2; + GstClock *clock; init_data (&data); + clock = gst_system_clock_obtain (); rtpbin = gst_element_factory_make ("rtpbin", "rtpbin"); + gst_element_set_clock (rtpbin, clock); + gst_object_unref (clock); g_signal_connect (rtpbin, "pad-added", (GCallback) pad_added_cb, &data); g_signal_connect (rtpbin, "pad-removed", (GCallback) pad_removed_cb, &data); @@ -331,10 +347,14 @@ GST_START_TEST (test_cleanup_recv2) GstStateChangeReturn ret; GstFlowReturn res; gint count = 2; + GstClock *clock; init_data (&data); + clock = gst_system_clock_obtain (); rtpbin = gst_element_factory_make ("rtpbin", "rtpbin"); + gst_element_set_clock (rtpbin, clock); + gst_object_unref (clock); g_signal_connect (rtpbin, "pad-added", (GCallback) pad_added_cb, &data); g_signal_connect (rtpbin, "pad-removed", (GCallback) pad_removed_cb, &data); @@ -441,7 +461,7 @@ GST_START_TEST (test_request_pad_by_template_name) GST_END_TEST; static GstElement * -encoder_cb (GstElement * rtpbin, guint sessid, GstElement * bin) +encoder_cb (G_GNUC_UNUSED GstElement * rtpbin, guint sessid, GstElement * bin) { GstPad *srcpad, *sinkpad; @@ -458,7 +478,7 @@ encoder_cb (GstElement * rtpbin, guint sessid, GstElement * bin) } static GstElement * -encoder_cb2 (GstElement * rtpbin, guint sessid, GstElement * bin) +encoder_cb2 (G_GNUC_UNUSED GstElement * rtpbin, guint sessid, GstElement * bin) { GstPad *srcpad, *sinkpad; @@ -518,7 +538,8 @@ GST_START_TEST (test_encoder) GST_END_TEST; static GstElement * -decoder_cb (GstElement * rtpbin, guint sessid, gpointer user_data) +decoder_cb (G_GNUC_UNUSED GstElement * rtpbin, G_GNUC_UNUSED guint sessid, + G_GNUC_UNUSED gpointer user_data) { GstElement *bin; GstPad *srcpad, *sinkpad; @@ -574,7 +595,8 @@ GST_START_TEST (test_decoder) GST_END_TEST; static GstElement * -aux_sender_cb (GstElement * rtpbin, guint sessid, gpointer user_data) +aux_sender_cb (G_GNUC_UNUSED GstElement * rtpbin, G_GNUC_UNUSED guint sessid, + gpointer user_data) { GstElement *bin; GstPad *srcpad, *sinkpad; @@ -652,7 +674,8 @@ GST_START_TEST (test_aux_sender) GST_END_TEST; static GstElement * -aux_receiver_cb (GstElement * rtpbin, guint sessid, gpointer user_data) +aux_receiver_cb (G_GNUC_UNUSED GstElement * rtpbin, G_GNUC_UNUSED guint sessid, + G_GNUC_UNUSED gpointer user_data) { GstElement *bin; GstPad *srcpad, *sinkpad; @@ -941,8 +964,11 @@ _pad_added (G_GNUC_UNUSED GstElement * rtpbin, GstPad * pad, GstHarness * h) GST_START_TEST (test_quick_shutdown) { guint r; + guint repeats = 1000; + if (RUNNING_ON_VALGRIND) + repeats = 2; - for (r = 0; r < 1000; r++) { + for (r = 0; r < repeats; r++) { guint i; GstHarness *h = gst_harness_new_with_padnames ("rtpbin", "recv_rtp_sink_0", NULL); @@ -969,6 +995,81 @@ GST_START_TEST (test_quick_shutdown) GST_END_TEST; +static GstBuffer * +generate_rtcp_sr_buffer (guint ssrc) +{ + GstBuffer *buf; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + GstRTCPPacket packet; + + buf = gst_rtcp_buffer_new (1000); + fail_unless (gst_rtcp_buffer_map (buf, GST_MAP_READWRITE, &rtcp)); + fail_unless (gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_SR, &packet)); + gst_rtcp_packet_sr_set_sender_info (&packet, ssrc, 0, 0, 1, 1); + gst_rtcp_buffer_unmap (&rtcp); + return buf; +} + +typedef struct +{ + GstHarness *h_rtp; + GstHarness *h_rtcp; +} PushRTCPSRCtx; + +static gpointer +_push_sr_buffers (gpointer user_data) +{ + PushRTCPSRCtx *ctx = user_data; + + gst_harness_set_src_caps (ctx->h_rtcp, + gst_caps_new_empty_simple ("application/x-rtcp")); + gst_harness_push (ctx->h_rtcp, generate_rtcp_sr_buffer (1111)); + return NULL; +} + +static gpointer +_push_rtp_buffers (gpointer user_data) +{ + PushRTCPSRCtx *ctx = user_data; + + g_usleep (10); + + gst_harness_set_src_caps (ctx->h_rtp, + gst_caps_new_simple ("application/x-rtp", + "clock-rate", G_TYPE_INT, 8000, "payload", G_TYPE_INT, 100, NULL)); + gst_harness_push (ctx->h_rtp, generate_rtp_buffer (0, 0, 0, 100, 1111)); + return NULL; +} + +GST_START_TEST (test_recv_rtp_and_rtcp_simultaneously) +{ + guint r; + guint repeats = 1000; + if (RUNNING_ON_VALGRIND) + repeats = 2; + + for (r = 0; r < repeats; r++) { + PushRTCPSRCtx ctx; + GThread *t0; + GThread *t1; + + ctx.h_rtcp = gst_harness_new_with_padnames ("rtpbin", + "recv_rtcp_sink_0", NULL); + ctx.h_rtp = gst_harness_new_with_element (ctx.h_rtcp->element, + "recv_rtp_sink_0", NULL); + + t0 = g_thread_new ("push sr", _push_sr_buffers, &ctx); + t1 = g_thread_new ("push rtp", _push_rtp_buffers, &ctx); + + g_thread_join (t0); + g_thread_join (t1); + gst_harness_teardown (ctx.h_rtp); + gst_harness_teardown (ctx.h_rtcp); + } +} + +GST_END_TEST; + static Suite * rtpbin_suite (void) { @@ -987,6 +1088,7 @@ rtpbin_suite (void) tcase_add_test (tc_chain, test_aux_receiver); tcase_add_test (tc_chain, test_sender_eos); tcase_add_test (tc_chain, test_quick_shutdown); + tcase_add_test (tc_chain, test_recv_rtp_and_rtcp_simultaneously); return s; } From d602691bc416783b239f12f5097763770e51a5c5 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 1 Sep 2021 14:59:28 +0200 Subject: [PATCH 064/339] rtpmanager: add header-extension for Region of Interest. It will take the GstVideoRegionOfInterestMeta, and turn one of its specified types into a RTP header-extension. Co-authored-by: Camilo Celis Guzman --- .../gst/rtpmanager/gstrtphdrext-roi.c | 240 +++++++ .../gst/rtpmanager/gstrtphdrext-roi.h | 34 + .../gst/rtpmanager/gstrtpmanager.c | 2 + .../gst/rtpmanager/meson.build | 1 + .../tests/check/elements/rtphdrext-roi.c | 632 ++++++++++++++++++ .../gst-plugins-good/tests/check/meson.build | 2 +- 6 files changed, 910 insertions(+), 1 deletion(-) create mode 100644 subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c create mode 100644 subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.h create mode 100644 subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c new file mode 100644 index 00000000000..a2d6968a08a --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c @@ -0,0 +1,240 @@ +/* GStreamer + * Copyright (C) <2021> Havard Graff + * Camilo Celis Guzman + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more + */ + +/** + * SECTION:element-rtphdrextroi + * @title: rtphdrextroi + * @short_description: Region of interest (ROI) RTP Header Extension + * + * + * Since: 1.20 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstrtphdrext-roi.h" +#include + +/* + * Internal roi-ext-hdr ID used as a safety mechanism, to make sure + * that we not only only care about a specific roi_type (as in + * GstVideoRegionOfInterestMeta) but also that it has been payloaded + * and payloaded using the set of callbacks implemented below. + */ +#define ROI_EXTHDR_TYPE 0xFD + +#define ROI_HDR_EXT_URI GST_RTP_HDREXT_BASE"TBD:draft-ford-avtcore-roi-extension-00" + +enum +{ + PROP_0, + PROP_ROI_TYPE, +}; + +struct _GstRTPHeaderExtensionRoi +{ + GstRTPHeaderExtension parent; + + guint roi_type; +}; + +G_DEFINE_TYPE_WITH_CODE (GstRTPHeaderExtensionRoi, + gst_rtp_header_extension_roi, GST_TYPE_RTP_HEADER_EXTENSION, + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "rtphdrextroi", 0, + "RTP ROI Header Extensions")); +GST_ELEMENT_REGISTER_DEFINE (rtphdrextroi, "rtphdrextroi", + GST_RANK_MARGINAL, GST_TYPE_RTP_HEADER_EXTENSION_ROI); + +static void +gst_rtp_header_extension_roi_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (object); + switch (prop_id) { + case PROP_ROI_TYPE: + g_value_set_uint (value, self->roi_type); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_rtp_header_extension_roi_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (object); + switch (prop_id) { + case PROP_ROI_TYPE: + self->roi_type = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GstRTPHeaderExtensionFlags +gst_rtp_header_extension_roi_get_supported_flags (GstRTPHeaderExtension * ext) +{ + return GST_RTP_HEADER_EXTENSION_ONE_BYTE; +} + +static gsize +gst_rtp_header_extension_roi_get_max_size (GstRTPHeaderExtension * ext, + const GstBuffer * input_meta) +{ + return 11; +} + + +static gssize +gst_rtp_header_extension_roi_write (GstRTPHeaderExtension * ext, + const GstBuffer * input_meta, GstRTPHeaderExtensionFlags write_flags, + G_GNUC_UNUSED GstBuffer * output, guint8 * data, gsize size) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); + GstVideoRegionOfInterestMeta *meta; + gpointer state = NULL; + + g_return_val_if_fail (size >= + gst_rtp_header_extension_roi_get_max_size (ext, NULL), -1); + g_return_val_if_fail (write_flags & + gst_rtp_header_extension_roi_get_supported_flags (ext), -1); + + while ((meta = (GstVideoRegionOfInterestMeta *) + gst_buffer_iterate_meta_filtered ((GstBuffer *) input_meta, &state, + GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE))) { + GstStructure *extra_param_s; + guint num_faces = 0; + + /* we only really care to write RoI metas coming from pexfdbin + * so we filter also by GstVideoRegionOfInterestMeta->roi_type */ + if (self->roi_type != meta->roi_type) + continue; + + // FIXME: Issue 23027 + // If these asserts got hit the problem either in the scaler of in crop element + // g_assert (meta->w > 0); + // g_assert (meta->h > 0); + + /* RoI coordinates */ + GST_WRITE_UINT16_BE (&data[0], meta->x); + GST_WRITE_UINT16_BE (&data[2], meta->y); + GST_WRITE_UINT16_BE (&data[4], meta->w); + GST_WRITE_UINT16_BE (&data[6], meta->h); + + /* RoI type (as in the roi-ext-hdr type) */ + GST_WRITE_UINT8 (&data[8], ROI_EXTHDR_TYPE); + + /* RoI - number of faces */ + + extra_param_s = + gst_video_region_of_interest_meta_get_param (meta, "extra-param"); + if (extra_param_s) + gst_structure_get_uint (extra_param_s, "num_faces", &num_faces); + GST_WRITE_UINT16_BE (&data[9], num_faces); + + return 11; + } + + return 0; +} + +static gboolean +gst_rtp_header_extension_roi_read (GstRTPHeaderExtension * ext, + GstRTPHeaderExtensionFlags read_flags, const guint8 * data, gsize size, + GstBuffer * buffer) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); + guint16 roi_type = GST_READ_UINT8 (&data[8]); + + /* safety mechanism to only consider those roi-ext-hdr types + * which we care about */ + if (roi_type == ROI_EXTHDR_TYPE) { + guint16 x = GST_READ_UINT16_BE (&data[0]); + guint16 y = GST_READ_UINT16_BE (&data[2]); + guint16 w = GST_READ_UINT16_BE (&data[4]); + guint16 h = GST_READ_UINT16_BE (&data[6]); + // guint16 roi_type = GST_READ_UINT8 (&bytes[8]); + guint16 num_faces = GST_READ_UINT16_BE (&data[9]); + + GstVideoRegionOfInterestMeta *meta = + gst_buffer_add_video_region_of_interest_meta_id (buffer, + self->roi_type, x, y, w, h); + + GstStructure *extra_param_s = gst_structure_new ("extra-param", + "num_faces", G_TYPE_UINT, num_faces, NULL); + // FIXME: Issue 23027 + // If these asserts got hit the problem in header corruption + // Need to look at re-transmit (?) + // g_assert (w > 0); + // g_assert (h > 0); + gst_video_region_of_interest_meta_add_param (meta, extra_param_s); + + } + + return TRUE; +} + +static void +gst_rtp_header_extension_roi_class_init (GstRTPHeaderExtensionRoiClass * klass) +{ + GstRTPHeaderExtensionClass *rtp_hdr_class; + GstElementClass *gstelement_class; + GObjectClass *gobject_class; + + rtp_hdr_class = GST_RTP_HEADER_EXTENSION_CLASS (klass); + gobject_class = (GObjectClass *) klass; + gstelement_class = GST_ELEMENT_CLASS (klass); + + gobject_class->get_property = gst_rtp_header_extension_roi_get_property; + gobject_class->set_property = gst_rtp_header_extension_roi_set_property; + + /** + * rtphdrextroi:roi-type: + * + * What roi-type (GQuark) to write the extension-header for. + * + * Since: 1.20 + */ + g_object_class_install_property (gobject_class, PROP_ROI_TYPE, + g_param_spec_uint ("roi-type", "ROI TYPE", + "What roi-type (GQuark) to write the extension-header for", + 0, G_MAXUINT32, 0, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + rtp_hdr_class->get_supported_flags = + gst_rtp_header_extension_roi_get_supported_flags; + rtp_hdr_class->get_max_size = gst_rtp_header_extension_roi_get_max_size; + rtp_hdr_class->write = gst_rtp_header_extension_roi_write; + rtp_hdr_class->read = gst_rtp_header_extension_roi_read; + + gst_element_class_set_static_metadata (gstelement_class, + "Region-of-Interest (ROI) RTP Header Extension", + GST_RTP_HDREXT_ELEMENT_CLASS, + "Region-of-Interest (ROI) RTP Header Extension", + "Havard Graff "); + gst_rtp_header_extension_class_set_uri (rtp_hdr_class, ROI_HDR_EXT_URI); +} + +static void +gst_rtp_header_extension_roi_init (GstRTPHeaderExtensionRoi * self) +{ + GST_DEBUG_OBJECT (self, "creating element"); +} diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.h new file mode 100644 index 00000000000..ba7a93e8849 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.h @@ -0,0 +1,34 @@ +/* GStreamer + * Copyright (C) <2021> Havard Graff + * Camilo Celis Guzman + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more + */ + +#ifndef __GST_RTPHDREXT_ROI_H__ +#define __GST_RTPHDREXT_ROI_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_RTP_HEADER_EXTENSION_ROI (gst_rtp_header_extension_roi_get_type()) + +G_DECLARE_FINAL_TYPE (GstRTPHeaderExtensionRoi, gst_rtp_header_extension_roi, GST, RTP_HEADER_EXTENSION_ROI, GstRTPHeaderExtension) + +GST_ELEMENT_REGISTER_DECLARE (rtphdrextroi); + +#define GST_RTP_HEADER_EXTENSION_ROI_CAST(obj) ((GstRTPHeaderExtensionRoi *)obj) + +G_END_DECLS + +#endif /* __GST_RTPHDREXT_ROI_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmanager.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmanager.c index 59a9eeb3a2c..d93aa310417 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmanager.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmanager.c @@ -40,6 +40,7 @@ #include "gstrtphdrext-ntp.h" #include "gstrtphdrext-repairedstreamid.h" #include "gstrtphdrext-streamid.h" +#include "gstrtphdrext-roi.h" static gboolean plugin_init (GstPlugin * plugin) @@ -65,6 +66,7 @@ plugin_init (GstPlugin * plugin) ret |= GST_ELEMENT_REGISTER (rtphdrextntp64, plugin); ret |= GST_ELEMENT_REGISTER (rtphdrextstreamid, plugin); ret |= GST_ELEMENT_REGISTER (rtphdrextrepairedstreamid, plugin); + ret |= GST_ELEMENT_REGISTER (rtphdrextroi, plugin); return ret; } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/meson.build b/subprojects/gst-plugins-good/gst/rtpmanager/meson.build index 11435d902f0..20c95f742c9 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/meson.build +++ b/subprojects/gst-plugins-good/gst/rtpmanager/meson.build @@ -9,6 +9,7 @@ rtpmanager_sources = [ 'gstrtphdrext-ntp.c', 'gstrtphdrext-repairedstreamid.c', 'gstrtphdrext-streamid.c', + 'gstrtphdrext-roi.c', 'gstrtpmux.c', 'gstrtpptdemux.c', 'gstrtprtxqueue.c', diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c b/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c new file mode 100644 index 00000000000..f805415a1f2 --- /dev/null +++ b/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c @@ -0,0 +1,632 @@ +/* GStreamer + * + * unit test for rtphdrext-roi elements + * + * Copyright (C) 2020-2021 Pexip AS. + * @author: Camilo Celis Guzman + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include + +#define URI GST_RTP_HDREXT_BASE "TBD:draft-ford-avtcore-roi-extension-00" +#define EXTMAP_ID 10 + +#define SRC_CAPS_STR "video/x-raw,format=I420" + +GST_START_TEST (test_rtphdrext_roi_basic) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = gst_caps_from_string (SRC_CAPS_STR); + gst_harness_set_src_caps (h, src_caps); + + gst_harness_add_src_parse (h, "videotestsrc ! " + "capsfilter caps=\"" SRC_CAPS_STR "\"", FALSE); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + gst_harness_play (h); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push_from_src (h)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_harness_teardown (h); +} + +GST_END_TEST; + +static GstCaps * +_i420_caps (gint width, gint height) +{ + GstVideoInfo info; + gst_video_info_init (&info); + gst_video_info_set_format (&info, GST_VIDEO_FORMAT_I420, width, height); + GST_VIDEO_INFO_FPS_N (&info) = 1; + GST_VIDEO_INFO_FPS_D (&info) = 1; + GST_VIDEO_INFO_PAR_N (&info) = 1; + GST_VIDEO_INFO_PAR_D (&info) = 1; + return gst_video_info_to_caps (&info); +} + +static GstBuffer * +_gst_harness_create_raw_buffer (GstHarness * h, GstCaps * caps) +{ + GstBuffer *buf; + GstVideoInfo video_info; + + gst_video_info_init (&video_info); + gst_video_info_from_caps (&video_info, caps); + + buf = gst_harness_create_buffer (h, GST_VIDEO_INFO_SIZE (&video_info)); + gst_buffer_memset (buf, 0, 0, GST_VIDEO_INFO_SIZE (&video_info)); + fail_unless (buf); + + gst_buffer_add_video_meta_full (buf, + GST_VIDEO_FRAME_FLAG_NONE, + GST_VIDEO_INFO_FORMAT (&video_info), + GST_VIDEO_INFO_WIDTH (&video_info), + GST_VIDEO_INFO_HEIGHT (&video_info), + GST_VIDEO_INFO_N_PLANES (&video_info), + video_info.offset, video_info.stride); + + GST_BUFFER_PTS (buf) = 0; + GST_BUFFER_DURATION (buf) = GST_SECOND / 30; + + return buf; +} + +GST_START_TEST (test_rtphdrext_roi_default_roi_type) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GstMeta *meta; + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + const GQuark default_roi_type = 0; + GQuark pay_roi_type = ~default_roi_type; + GQuark depay_roi_type = ~default_roi_type; + + gpointer state = NULL; + gint num_roi_metas_found = 0; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + g_object_get (pay_ext, "roi-type", &pay_roi_type, NULL); + g_object_get (depay_ext, "roi-type", &depay_roi_type, NULL); + fail_unless_equals_int (pay_roi_type, default_roi_type); + fail_unless_equals_int (depay_roi_type, default_roi_type); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + /* add a RoI meta on an input buffer with the default RoI type + * and expect this to be payloaded and depayloaded back as a RoI meta on + * the output buffer */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + default_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly one RoI meta on the output buffer with + * the expected RoI coordinates and type */ + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int ((gint) default_roi_type, (gint) vmeta->roi_type); + fail_unless_equals_int (0, vmeta->x); + fail_unless_equals_int (0, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + } + } + fail_unless_equals_int (1, num_roi_metas_found); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_unknown_roi_type) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GQuark other_roi_type = g_quark_from_string ("SomeRoITypeStr"); + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + /* add a RoI meta on an input buffer with an unknown RoI type and + * don't expect this to be payloaded and depayloaded as a RoI meta on + * the output buffer as we strictly only payload known RoI types */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_explicit_roi_type) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GstMeta *meta; + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + const GQuark other_roi_type = 0xFFFFFFFF; + + gpointer state = NULL; + gint num_roi_metas_found = 0; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + gst_buffer_unref (gst_harness_pull (h)); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* explicitly set a roi-type on both payloader and depayloader extensions + * and expect a buffer with such roi-type added to their RoI meta to be + * correctly payloaded and depayloaded */ + g_object_set (pay_ext, "roi-type", other_roi_type, NULL); + g_object_set (depay_ext, "roi-type", other_roi_type, NULL); + + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly one RoI meta on the output buffer with + * the expected RoI coordinates and type */ + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int ((gint) other_roi_type, (gint) vmeta->roi_type); + fail_unless_equals_int (0, vmeta->x); + fail_unless_equals_int (0, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + } + } + fail_unless_equals_int (1, num_roi_metas_found); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_pay_only) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GstMeta *meta; + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + const GQuark default_roi_type = 0; + const GQuark other_roi_type = ~default_roi_type; + + gpointer state = NULL; + gint num_roi_metas_found = 0; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + /* set a roi-type on the payloader only and expect this to be + * partially correctly depayloaded as the depayloader will add a meta for + * ANY HDREXT-ROI that has been payloaded by a GStreamer payloader with + * the added rtphdrext-roi element. However, the roi-type on the RoI meta + * would be the default one for the depayloader */ + g_object_set (pay_ext, "roi-type", other_roi_type, NULL); + + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly one RoI meta on the output buffer with + * the expected RoI coordinates and type */ + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int ((gint) default_roi_type, (gint) vmeta->roi_type); + fail_unless_equals_int (0, vmeta->x); + fail_unless_equals_int (0, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + } + } + fail_unless_equals_int (1, num_roi_metas_found); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_depay_only) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + const GQuark default_roi_type = 0; + const GQuark other_roi_type = ~default_roi_type; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + /* set a roi-type on the depayloader only */ + g_object_set (depay_ext, "roi-type", other_roi_type, NULL); + + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get no metas as we would not be able to payload an + * unknown type hence no RTP header extension is added and the depayloader + * wont, ofcourse add any RoI metas on the output buffer */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + + +static Suite * +rtphdrext_roi_suite (void) +{ + Suite *s = suite_create ("rtphdrext_roi"); + TCase *tc_chain = tcase_create ("general"); + + suite_add_tcase (s, tc_chain); + + tcase_add_test (tc_chain, test_rtphdrext_roi_basic); + tcase_add_test (tc_chain, test_rtphdrext_roi_default_roi_type); + tcase_add_test (tc_chain, test_rtphdrext_roi_unknown_roi_type); + tcase_add_test (tc_chain, test_rtphdrext_roi_explicit_roi_type); + tcase_add_test (tc_chain, test_rtphdrext_roi_explicit_roi_type_pay_only); + tcase_add_test (tc_chain, test_rtphdrext_roi_explicit_roi_type_depay_only); + + return s; +} + +GST_CHECK_MAIN (rtphdrext_roi) diff --git a/subprojects/gst-plugins-good/tests/check/meson.build b/subprojects/gst-plugins-good/tests/check/meson.build index 3a4b5c99a67..4b29b16bb31 100644 --- a/subprojects/gst-plugins-good/tests/check/meson.build +++ b/subprojects/gst-plugins-good/tests/check/meson.build @@ -87,6 +87,7 @@ good_tests = [ if not get_option('rtp').disabled() and not get_option('rtpmanager').disabled() good_tests += [ [ 'elements/rtphdrext-colorspace' ], + [ 'elements/rtphdrext-roi' ], [ 'elements/rtph261', ], [ 'elements/rtph263' ], [ 'elements/rtph264' ], @@ -103,7 +104,6 @@ if not get_option('rtp').disabled() and not get_option('rtpmanager').disabled() [ 'elements/rtphdrextsdes', false, [gstrtp_dep, gstsdp_dep] ], [ 'elements/rtpjitterbuffer' ], [ 'elements/rtpjpeg' ], - [ 'elements/rtptimerqueue', false, [gstrtp_dep], ['../../gst/rtpmanager/rtptimerqueue.c']], From c69b241978198953ba0ff2dcb867f24e0785b43b Mon Sep 17 00:00:00 2001 From: David Buchmann Date: Fri, 3 Oct 2014 11:29:11 +0200 Subject: [PATCH 065/339] rtmpsrc: Expose bandwidth info from librtmp --- .../gst-plugins-bad/ext/rtmp/gstrtmpsrc.c | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c index f69b641b2d9..626474d0643 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c @@ -80,9 +80,12 @@ enum { PROP_0, PROP_LOCATION, - PROP_TIMEOUT + PROP_TIMEOUT, + PROP_BYTES_IN, + PROP_CLIENT_BW, + PROP_SERVER_BW, #if 0 - PROP_SWF_URL, + PROP_SWF_URL, PROP_PAGE_URL #endif }; @@ -147,6 +150,21 @@ gst_rtmp_src_class_init (GstRTMPSrcClass * klass) 0, G_MAXINT, DEFAULT_TIMEOUT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_BYTES_IN, + g_param_spec_int ("bytes-in", "Bytes in", + "Number of bytes received (-1 = not available)", -1, G_MAXINT, -1, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_CLIENT_BW, + g_param_spec_int ("client-bandwidth", "Client bandwidth", + "Client bandwidth (-1 = not available)", -1, G_MAXINT, -1, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_SERVER_BW, + g_param_spec_int ("server-bandwidth", "Server bandwidth", + "Server bandwidth (-1 = not available)", -1, G_MAXINT, -1, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + gst_element_class_add_static_pad_template (gstelement_class, &srctemplate); gst_element_class_set_static_metadata (gstelement_class, @@ -319,6 +337,24 @@ gst_rtmp_src_get_property (GObject * object, guint prop_id, GValue * value, case PROP_TIMEOUT: g_value_set_int (value, src->timeout); break; + case PROP_BYTES_IN: + if (src->rtmp) + g_value_set_int (value, src->rtmp->m_nBytesIn); + else + g_value_set_int (value, -1); + break; + case PROP_CLIENT_BW: + if (src->rtmp) + g_value_set_int (value, src->rtmp->m_nClientBW); + else + g_value_set_int (value, -1); + break; + case PROP_SERVER_BW: + if (src->rtmp) + g_value_set_int (value, src->rtmp->m_nServerBW); + else + g_value_set_int (value, -1); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; From 7bfc4b05fdcbce02f70546ea12680584b0d8ac0f Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 19 Nov 2015 15:17:05 +1100 Subject: [PATCH 066/339] rtmpsrc: use old code for unlock The problem with calling RTMP_Close in unlock, is that this can (and do) cause MT-issues with an ongoing call to RTMP_Read in _create. There is also no way you could protect this with a mutex, since the main purpose of _unlock in RTMP is to stop an ongoing (potentially long blocking) call to RTMP_Read, and mutexing them against eachother then misses the point. By simply doing a shutdown on the socket, you are MT-safe in terms of what RTMP_Read might be doing, as well as instantly unblocking a potential tcp-timeout on a server that has gone missing etc. --- subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c index 626474d0643..9e9b2238a07 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c @@ -61,6 +61,7 @@ #include #include #include +#include #include @@ -702,10 +703,11 @@ gst_rtmp_src_unlock (GstBaseSrc * basesrc) GST_DEBUG_OBJECT (rtmpsrc, "unlock"); - /* This closes the socket, which means that any pending socket calls - * error out. */ - if (rtmpsrc->rtmp) { - RTMP_Close (rtmpsrc->rtmp); + /* This cancels the recv() underlying RTMP_Read, but will cause a + * SIGPIPE. Hopefully the app is ignoring it, or you've patched + * librtmp. */ + if (rtmpsrc->rtmp && rtmpsrc->rtmp->m_sb.sb_socket > 0) { + shutdown (rtmpsrc->rtmp->m_sb.sb_socket, SHUT_RDWR); } return TRUE; From 891d79ec423bb074db79e78c2b3a3b57c76b4cec Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 4 Dec 2015 00:48:58 +1100 Subject: [PATCH 067/339] rtmpsrc: plug leaks in error-scenarios --- subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c index 9e9b2238a07..8fd2d69c7a8 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c @@ -722,6 +722,7 @@ gst_rtmp_src_stop (GstBaseSrc * basesrc) src = GST_RTMP_SRC (basesrc); if (src->rtmp) { + RTMP_Close (src->rtmp); RTMP_Free (src->rtmp); src->rtmp = NULL; } From 2c0a466f2b675e5fea648d86a01cd69c528e51f2 Mon Sep 17 00:00:00 2001 From: Nico Cormier Date: Thu, 21 Jan 2016 15:39:47 +0100 Subject: [PATCH 068/339] rtmpsrc: Revert flow error for now, chasing #6051 --- subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c index 8fd2d69c7a8..6b236a25b58 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c @@ -467,7 +467,7 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) if (src->cur_offset == 0) { GST_ELEMENT_ERROR (src, RESOURCE, READ, (NULL), ("Failed to read any data from stream, check your URL")); - return GST_FLOW_ERROR; + return GST_FLOW_EOS; } else { GST_DEBUG_OBJECT (src, "Reading data gave EOS"); return GST_FLOW_EOS; From 3a1038954048fde1fde22911a75dcc8afe4dffd7 Mon Sep 17 00:00:00 2001 From: David Buchmann Date: Sat, 2 Apr 2016 18:36:11 +0200 Subject: [PATCH 069/339] rtmpsrc: Do not return FLOW_ERROR for connection that are closed --- subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c index 6b236a25b58..10db9d589c4 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c @@ -384,7 +384,7 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) if (!RTMP_IsConnected (src->rtmp)) { GST_DEBUG_OBJECT (src, "reconnecting"); if (!gst_rtmp_src_connect (src)) - return GST_FLOW_ERROR; + return GST_FLOW_EOS; } size = GST_BASE_SRC_CAST (pushsrc)->blocksize; From 30c2e8142a3eb28180d61f8265ada092769a80e4 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 2 Jun 2016 09:49:41 +0200 Subject: [PATCH 070/339] rtmpsrc: protect all access to the rtmp-object with the lock Or else it can and will crash. --- subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c index 10db9d589c4..7553111db49 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c @@ -437,11 +437,15 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) GST_BUFFER_OFFSET (buf) = src->cur_offset; src->cur_offset += size; - if (src->last_timestamp == GST_CLOCK_TIME_NONE) - src->last_timestamp = src->rtmp->m_mediaStamp * GST_MSECOND; - else - src->last_timestamp = - MAX (src->last_timestamp, src->rtmp->m_mediaStamp * GST_MSECOND); + RTMP_LOCK (src); + if (src->rtmp) { + if (src->last_timestamp == GST_CLOCK_TIME_NONE) + src->last_timestamp = src->rtmp->m_mediaStamp * GST_MSECOND; + else + src->last_timestamp = + MAX (src->last_timestamp, src->rtmp->m_mediaStamp * GST_MSECOND); + } + RTMP_UNLOCK (src); GST_LOG_OBJECT (src, "Created buffer of size %u at %" G_GINT64_FORMAT " with timestamp %" GST_TIME_FORMAT, size, GST_BUFFER_OFFSET (buf), From 0d5bfeb25d6abb5dd23d1e8e47387ea1a252abcc Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 1 Oct 2015 11:18:29 +0200 Subject: [PATCH 071/339] rtmpsink: add _unlock() and tests for it The main problem this patch is trying to solve, is that librtmp can poentially block a connection or a write for a very long time, potentially stalling the application for as long as a TCP timeout can last (30 sec) However, doing this in a MT safe way is much harder then it would appear. The "most common" scenario would be that librtmp is currently doing a write on the socket, and we terminate this write by using shutdown on that internal socket. Now, had we instead used RTMP_Close, we will often interfer with internal state inside librtmp as they are being used, so shutdown is our only option here. We need to add a close() as well, since the tests showed that even though we successfully cancelled one write, while doing a handshake there can be another one as well, and only by doing close are we preventing that second one from happening. By also protecting the internal state with a lock, we then use _trylock to figure out that we in fact are not allowed to touch the internal state, but then instead use shutdown... Finally, a looping wait is added for the case where the rtmp-lock has been taken, but the internals of librtmp have not yet allocated the socket it is about to work with, meaning our shutdown would fail silently, and we would still be waiting "forever". All of these changes are needed to make the supplied tests pass. Maybe an atomic counter could be more elegant then using lock and trylock, but it felt like the right solution, since we are both eliminating the internal state completely, and when not allowed to do so, using shutdown to break the wait. --- .../gst-plugins-bad/ext/rtmp/gstrtmpsink.c | 109 +++++++++++----- .../gst-plugins-bad/ext/rtmp/gstrtmpsink.h | 2 + tests/check/elements/rtmp.c | 116 ++++++++++++++++++ 3 files changed, 198 insertions(+), 29 deletions(-) create mode 100644 tests/check/elements/rtmp.c diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c index ec640ad0403..993a980e4c6 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c @@ -49,10 +49,16 @@ #endif #include +#include +#include GST_DEBUG_CATEGORY_STATIC (gst_rtmp_sink_debug); #define GST_CAT_DEFAULT gst_rtmp_sink_debug +#define RTMP_LOCK(sink) g_mutex_lock (&(sink)->rtmp_lock) +#define RTMP_UNLOCK(sink) g_mutex_unlock (&(sink)->rtmp_lock) +#define RTMP_TRYLOCK(sink) g_mutex_trylock (&(sink)->rtmp_lock) + #define DEFAULT_LOCATION NULL enum @@ -74,6 +80,7 @@ static void gst_rtmp_sink_set_property (GObject * object, guint prop_id, static void gst_rtmp_sink_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_rtmp_sink_finalize (GObject * object); +static gboolean gst_rtmp_sink_unlock (GstBaseSink * sink); static gboolean gst_rtmp_sink_stop (GstBaseSink * sink); static gboolean gst_rtmp_sink_start (GstBaseSink * sink); static gboolean gst_rtmp_sink_event (GstBaseSink * sink, GstEvent * event); @@ -116,6 +123,7 @@ gst_rtmp_sink_class_init (GstRTMPSinkClass * klass) gstbasesink_class->start = GST_DEBUG_FUNCPTR (gst_rtmp_sink_start); gstbasesink_class->stop = GST_DEBUG_FUNCPTR (gst_rtmp_sink_stop); + gstbasesink_class->unlock = GST_DEBUG_FUNCPTR (gst_rtmp_sink_unlock); gstbasesink_class->render = GST_DEBUG_FUNCPTR (gst_rtmp_sink_render); gstbasesink_class->set_caps = GST_DEBUG_FUNCPTR (gst_rtmp_sink_setcaps); gstbasesink_class->event = GST_DEBUG_FUNCPTR (gst_rtmp_sink_event); @@ -137,6 +145,8 @@ gst_rtmp_sink_init (GstRTMPSink * sink) GST_ERROR_OBJECT (sink, "WSAStartup failed: 0x%08x", WSAGetLastError ()); } #endif + + g_mutex_init (&sink->rtmp_lock); } static void @@ -148,6 +158,7 @@ gst_rtmp_sink_finalize (GObject * object) WSACleanup (); #endif g_free (sink->uri); + g_mutex_clear (&sink->rtmp_lock); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -186,6 +197,7 @@ gst_rtmp_sink_start (GstBaseSink * basesink) sink->first = TRUE; sink->have_write_error = FALSE; + sink->connecting = FALSE; return TRUE; @@ -199,6 +211,44 @@ gst_rtmp_sink_start (GstBaseSink * basesink) return FALSE; } +static gboolean +gst_rtmp_sink_unlock (GstBaseSink * basesink) +{ + GstRTMPSink *sink = GST_RTMP_SINK (basesink); + + if (sink->rtmp == NULL) + return TRUE; + + /* Check to see if we currently are doing any activity towards librtmp */ + GST_DEBUG_OBJECT (sink, "Trying to lock"); + + if (!RTMP_TRYLOCK (sink)) { + GST_DEBUG_OBJECT (sink, "Lock NOT aquired..."); + /* if we are trying to connect, but the internal socket are not yet + initialized, we keep trying until either connection have failed or + the socket comes up */ + while (sink->connecting && sink->rtmp->m_sb.sb_socket == -1) { + g_thread_yield (); + } + + if (sink->rtmp->m_sb.sb_socket >= 0) { + GST_DEBUG_OBJECT (sink, "Shutting down internal librtmp socket"); + shutdown (sink->rtmp->m_sb.sb_socket, SHUT_RDWR); + close (sink->rtmp->m_sb.sb_socket); + } + + } else { + GST_DEBUG_OBJECT (sink, "Lock aquired..."); + RTMP_Close (sink->rtmp); + RTMP_Free (sink->rtmp); + sink->rtmp = NULL; + RTMP_UNLOCK (sink); + } + + return TRUE; +} + + static gboolean gst_rtmp_sink_stop (GstBaseSink * basesink) { @@ -228,30 +278,34 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) gboolean need_unref = FALSE; GstMapInfo map = GST_MAP_INFO_INIT; - if (sink->rtmp == NULL) { - /* Do not crash */ - GST_ELEMENT_ERROR (sink, RESOURCE, WRITE, (NULL), ("Failed to write data")); - return GST_FLOW_ERROR; - } - /* Ignore buffers that are in the stream headers (caps) */ if (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_HEADER)) { return GST_FLOW_OK; } + if (sink->have_write_error) + goto write_failed; + if (sink->first) { /* open the connection */ - if (!RTMP_IsConnected (sink->rtmp)) { - if (!RTMP_Connect (sink->rtmp, NULL)) - goto connection_failed; + sink->connecting = TRUE; + RTMP_LOCK (sink); - if (!RTMP_ConnectStream (sink->rtmp, 0)) { - RTMP_Close (sink->rtmp); + if (sink->rtmp == NULL) { + RTMP_UNLOCK (sink); + goto connection_failed; + } + if (!RTMP_IsConnected (sink->rtmp)) { + if (!RTMP_Connect (sink->rtmp, NULL) + || !RTMP_ConnectStream (sink->rtmp, 0)) { + RTMP_UNLOCK (sink); goto connection_failed; } GST_DEBUG_OBJECT (sink, "Opened connection to %s", sink->rtmp_uri); } + RTMP_UNLOCK (sink); + sink->connecting = FALSE; /* Prepend the header from the caps to the first non header buffer */ if (sink->header) { @@ -259,20 +313,20 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) gst_buffer_ref (buf)); need_unref = TRUE; } - sink->first = FALSE; } - if (sink->have_write_error) - goto write_failed; - GST_LOG_OBJECT (sink, "Sending %" G_GSIZE_FORMAT " bytes to RTMP server", gst_buffer_get_size (buf)); gst_buffer_map (buf, &map, GST_MAP_READ); - if (RTMP_Write (sink->rtmp, (char *) map.data, map.size) <= 0) + RTMP_LOCK (sink); + if (sink->rtmp && RTMP_Write (sink->rtmp, (char *) map.data, map.size) <= 0) { + RTMP_UNLOCK (sink); goto write_failed; + } + RTMP_UNLOCK (sink); gst_buffer_unmap (buf, &map); if (need_unref) @@ -298,6 +352,7 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) RTMP_Free (sink->rtmp); sink->rtmp = NULL; g_free (sink->rtmp_uri); + sink->connecting = FALSE; sink->rtmp_uri = NULL; sink->have_write_error = TRUE; @@ -414,25 +469,21 @@ gst_rtmp_sink_setcaps (GstBaseSink * sink, GstCaps * caps) GST_DEBUG_OBJECT (sink, "caps set to %" GST_PTR_FORMAT, caps); + s = gst_caps_get_structure (caps, 0); + sh = gst_structure_get_value (s, "streamheader"); + if (sh == NULL) { + GST_DEBUG_OBJECT (rtmpsink, "No streamheader in caps"); + return TRUE; + } + /* Clear our current header buffer */ if (rtmpsink->header) { gst_buffer_unref (rtmpsink->header); rtmpsink->header = NULL; } - s = gst_caps_get_structure (caps, 0); - - sh = gst_structure_get_value (s, "streamheader"); - if (sh == NULL) - goto out; - - if (GST_VALUE_HOLDS_BUFFER (sh)) { - rtmpsink->header = gst_buffer_ref (gst_value_get_buffer (sh)); - } else if (GST_VALUE_HOLDS_ARRAY (sh)) { - GArray *buffers; - gint i; - - buffers = g_value_peek_pointer (sh); + rtmpsink->header = gst_buffer_new (); + buffers = g_value_peek_pointer (sh); /* Concatenate all buffers in streamheader into one */ rtmpsink->header = gst_buffer_new (); diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.h b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.h index daaa3bba774..c99c6acedf6 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.h +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.h @@ -52,6 +52,8 @@ struct _GstRTMPSink { RTMP *rtmp; gchar *rtmp_uri; /* copy of url for librtmp */ + GMutex rtmp_lock; + gboolean connecting; GstBuffer *header; gboolean first; diff --git a/tests/check/elements/rtmp.c b/tests/check/elements/rtmp.c new file mode 100644 index 00000000000..0615f1016e9 --- /dev/null +++ b/tests/check/elements/rtmp.c @@ -0,0 +1,116 @@ +/* GStreamer + * + * unit test for rtmp + * + * Copyright (C) <2016> Havard Graff + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include +#include + +#include +#include +#include + +static gint +add_listen_fd (int port) +{ + gint fd = socket (AF_INET6, SOCK_STREAM, 0); + g_assert_cmpint (fd, >=, 0); + + int sock_optval = 1; + setsockopt (fd, SOL_SOCKET, SO_REUSEADDR, &sock_optval, sizeof (sock_optval)); + + struct sockaddr_in6 sin; + memset (&sin, 0, sizeof (struct sockaddr_in6)); + sin.sin6_family = AF_INET6; + sin.sin6_port = htons (port); + sin.sin6_addr = in6addr_any; + + g_assert (bind (fd, (struct sockaddr *) &sin, sizeof (sin)) >= 0); + listen (fd, 10); + + return fd; +} + + +GST_START_TEST (rtmpsink_unlock) +{ + gint fd; + GstHarness *h; + GTimer *timer; + + fd = add_listen_fd (22000); + h = gst_harness_new_parse + ("queue ! rtmpsink location=rtmp://localhost:22000/app/streamname1"); + gst_harness_set_src_caps_str (h, "video/x-flv"); + + timer = g_timer_new (); + gst_harness_push (h, gst_buffer_new ()); + + if (__i__ == 1) + g_usleep (G_USEC_PER_SEC / 10); + + gst_harness_teardown (h); + fail_unless (g_timer_elapsed (timer, NULL) < 2.0); + + close (fd); +} + +GST_END_TEST; + +GST_START_TEST (rtmpsink_unlock_race) +{ + GstHarness *h = gst_harness_new_parse ("rtmpsink location=rtmp://a/b/c"); + GstHarnessThread *statechange, *push; + GstSegment segment; + GstCaps *caps = gst_caps_from_string ("video/x-flv"); + GstBuffer *buf = gst_buffer_new (); + + gst_segment_init (&segment, GST_FORMAT_TIME); + + statechange = gst_harness_stress_statechange_start_full (h, 1); + push = gst_harness_stress_push_buffer_start (h, caps, &segment, buf); + + g_usleep (G_USEC_PER_SEC * 1); + + gst_harness_stress_thread_stop (statechange); + gst_harness_stress_thread_stop (push); + + gst_caps_unref (caps); + gst_buffer_unref (buf); + gst_harness_teardown (h); +} + +GST_END_TEST; + + +static Suite * +rtmp_suite (void) +{ + Suite *s = suite_create ("rtmp"); + TCase *tc_chain = tcase_create ("general"); + suite_add_tcase (s, tc_chain); + + tcase_add_loop_test (tc_chain, rtmpsink_unlock, 0, 2); + tcase_add_test (tc_chain, rtmpsink_unlock_race); + + return s; +} + +GST_CHECK_MAIN (rtmp); From a28ff1cec4a7a6e8879d5f0aed69226271ad82f2 Mon Sep 17 00:00:00 2001 From: David Buchmann Date: Sat, 2 Apr 2016 18:42:51 +0200 Subject: [PATCH 072/339] rtmpsink: Port PROP_PACKETS_SENT from pre-1.0 merge so that management statistics has meaningful values --- subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c index 993a980e4c6..f98c45f39af 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c @@ -64,7 +64,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtmp_sink_debug); enum { PROP_0, - PROP_LOCATION + PROP_LOCATION, + PROP_PACKETS_SENT }; static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -114,6 +115,11 @@ gst_rtmp_sink_class_init (GstRTMPSinkClass * klass) g_param_spec_string ("location", "RTMP Location", "RTMP url", DEFAULT_LOCATION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PACKETS_SENT, + g_param_spec_int ("packets-sent", "Packets sent", + "Number of packets sent (-1 = not available)", -1, G_MAXINT, -1, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + gst_element_class_set_static_metadata (gstelement_class, "RTMP output sink", "Sink/Network", "Sends FLV content to a server via RTMP", @@ -537,6 +543,12 @@ gst_rtmp_sink_get_property (GObject * object, guint prop_id, case PROP_LOCATION: g_value_set_string (value, sink->uri); break; + case PROP_PACKETS_SENT: + if (sink->rtmp) + g_value_set_int (value, sink->rtmp->m_nPacketsSent); + else + g_value_set_int (value, -1); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; From f5788b5189a956865718ddc3fdfc6ae1e76fe6cb Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sun, 3 Apr 2016 20:32:05 +0200 Subject: [PATCH 073/339] rtmpsink: ease up on connect failures We don't like FLOW_ERROR in our pipes. --- .../gst-plugins-bad/ext/rtmp/gstrtmpsink.c | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c index f98c45f39af..ac66f9bd7c0 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c @@ -1,6 +1,7 @@ /* * GStreamer * Copyright (C) 2010 Jan Schmidt + * Copyright (C) 2016 Pexip * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -231,8 +232,8 @@ gst_rtmp_sink_unlock (GstBaseSink * basesink) if (!RTMP_TRYLOCK (sink)) { GST_DEBUG_OBJECT (sink, "Lock NOT aquired..."); /* if we are trying to connect, but the internal socket are not yet - initialized, we keep trying until either connection have failed or - the socket comes up */ + initialized, we keep trying until either connection have failed or + the socket comes up */ while (sink->connecting && sink->rtmp->m_sb.sb_socket == -1) { g_thread_yield (); } @@ -355,14 +356,9 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) { GST_ELEMENT_ERROR (sink, RESOURCE, OPEN_WRITE, (NULL), ("Could not connect to RTMP stream \"%s\" for writing", sink->uri)); - RTMP_Free (sink->rtmp); - sink->rtmp = NULL; - g_free (sink->rtmp_uri); sink->connecting = FALSE; - sink->rtmp_uri = NULL; sink->have_write_error = TRUE; - - return GST_FLOW_ERROR; + return GST_FLOW_OK; } } @@ -472,6 +468,8 @@ gst_rtmp_sink_setcaps (GstBaseSink * sink, GstCaps * caps) GstRTMPSink *rtmpsink = GST_RTMP_SINK (sink); GstStructure *s; const GValue *sh; + GArray *buffers; + gint i; GST_DEBUG_OBJECT (sink, "caps set to %" GST_PTR_FORMAT, caps); @@ -491,29 +489,22 @@ gst_rtmp_sink_setcaps (GstBaseSink * sink, GstCaps * caps) rtmpsink->header = gst_buffer_new (); buffers = g_value_peek_pointer (sh); - /* Concatenate all buffers in streamheader into one */ - rtmpsink->header = gst_buffer_new (); - for (i = 0; i < buffers->len; ++i) { - GValue *val; - GstBuffer *buf; + /* Concatenate all buffers in streamheader into one */ + for (i = 0; i < buffers->len; ++i) { + GValue *val; + GstBuffer *buf; - val = &g_array_index (buffers, GValue, i); - buf = g_value_peek_pointer (val); + val = &g_array_index (buffers, GValue, i); + buf = g_value_peek_pointer (val); - gst_buffer_ref (buf); + gst_buffer_ref (buf); - rtmpsink->header = gst_buffer_append (rtmpsink->header, buf); - } - } else { - GST_ERROR_OBJECT (rtmpsink, "streamheader field has unexpected type %s", - G_VALUE_TYPE_NAME (sh)); + rtmpsink->header = gst_buffer_append (rtmpsink->header, buf); } GST_DEBUG_OBJECT (rtmpsink, "have %" G_GSIZE_FORMAT " bytes of header data", gst_buffer_get_size (rtmpsink->header)); -out: - return TRUE; } From 1e742c4dce4dc4f25ecbb65f3a55fcc0fcb84f4e Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sun, 3 Apr 2016 20:33:45 +0200 Subject: [PATCH 074/339] rtmpsrc: allow connect and read to be interrupted. --- .../gst-plugins-bad/ext/rtmp/gstrtmpsrc.c | 138 +++++++++++------- .../gst-plugins-bad/ext/rtmp/gstrtmpsrc.h | 4 + tests/check/elements/rtmp.c | 25 ++++ 3 files changed, 114 insertions(+), 53 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c index 7553111db49..fc65ba47a7b 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c @@ -5,6 +5,7 @@ * 2002,2003 Colin Walters * 2001,2010 Bastien Nocera * 2010 Sebastian Dröge + * 2016 Pexip * * rtmpsrc.c: * @@ -62,6 +63,7 @@ #include #include #include +#include #include @@ -91,6 +93,10 @@ enum #endif }; +#define RTMP_LOCK(src) g_mutex_lock (&(src)->rtmp_lock) +#define RTMP_UNLOCK(src) g_mutex_unlock (&(src)->rtmp_lock) +#define RTMP_TRYLOCK(src) g_mutex_trylock (&(src)->rtmp_lock) + #define DEFAULT_LOCATION NULL #define DEFAULT_TIMEOUT 120 @@ -103,7 +109,6 @@ static void gst_rtmp_src_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_rtmp_src_finalize (GObject * object); -static gboolean gst_rtmp_src_connect (GstRTMPSrc * src); static gboolean gst_rtmp_src_unlock (GstBaseSrc * src); static gboolean gst_rtmp_src_stop (GstBaseSrc * src); static gboolean gst_rtmp_src_start (GstBaseSrc * src); @@ -203,6 +208,7 @@ gst_rtmp_src_init (GstRTMPSrc * rtmpsrc) rtmpsrc->last_timestamp = 0; rtmpsrc->timeout = DEFAULT_TIMEOUT; + g_mutex_init (&rtmpsrc->rtmp_lock); gst_base_src_set_format (GST_BASE_SRC (rtmpsrc), GST_FORMAT_TIME); } @@ -212,6 +218,7 @@ gst_rtmp_src_finalize (GObject * object) GstRTMPSrc *rtmpsrc = GST_RTMP_SRC (object); g_free (rtmpsrc->uri); + g_mutex_clear (&rtmpsrc->rtmp_lock); rtmpsrc->uri = NULL; #ifdef G_OS_WIN32 @@ -379,12 +386,25 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) src = GST_RTMP_SRC (pushsrc); - g_return_val_if_fail (src->rtmp != NULL, GST_FLOW_ERROR); + if (src->first) { + /* open the connection */ + src->first = FALSE; + src->connecting = TRUE; + RTMP_LOCK (src); - if (!RTMP_IsConnected (src->rtmp)) { - GST_DEBUG_OBJECT (src, "reconnecting"); - if (!gst_rtmp_src_connect (src)) - return GST_FLOW_EOS; + if (src->rtmp == NULL) { + RTMP_UNLOCK (src); + goto connect_error; + } + + if (!RTMP_IsConnected (src->rtmp)) { + if (!RTMP_Connect (src->rtmp, NULL)) { + RTMP_UNLOCK (src); + goto connect_error; + } + } + RTMP_UNLOCK (src); + src->connecting = FALSE; } size = GST_BASE_SRC_CAST (pushsrc)->blocksize; @@ -404,7 +424,11 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) bsize = 0; while (todo > 0) { - int read = RTMP_Read (src->rtmp, (char *) data, todo); + gint read = 0; + RTMP_LOCK (src); + if (src->rtmp) + read = RTMP_Read (src->rtmp, (char *) data, todo); + RTMP_UNLOCK (src); if (G_UNLIKELY (read == 0 && todo == size)) goto eos; @@ -457,6 +481,14 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) return GST_FLOW_OK; + /* ERRORS */ +connect_error: + { + GST_ERROR_OBJECT (src, "Could not connect to RTMP stream \"%s\" for reading", + src->uri); + src->connecting = FALSE; + return GST_FLOW_EOS; + } read_failed: { gst_buffer_unmap (buf, &map); @@ -604,24 +636,17 @@ gst_rtmp_src_do_seek (GstBaseSrc * basesrc, GstSegment * segment) return FALSE; } + src->discont = TRUE; + /* Initial seek */ if (src->cur_offset == 0 && segment->start == 0) - goto success; + return TRUE; if (!src->seekable) { GST_LOG_OBJECT (src, "Not a seekable stream"); return FALSE; } - /* If we have just disconnected in unlock(), we need to re-connect - * and also let librtmp read some data before sending a seek, - * otherwise it will stall. Calling create() does both. */ - if (!RTMP_IsConnected (src->rtmp)) { - GstBuffer *buffer = NULL; - gst_rtmp_src_create (GST_PUSH_SRC (basesrc), &buffer); - gst_buffer_replace (&buffer, NULL); - } - src->last_timestamp = GST_CLOCK_TIME_NONE; if (!RTMP_SendSeek (src->rtmp, segment->start / GST_MSECOND)) { GST_ERROR_OBJECT (src, "Seeking failed"); @@ -629,40 +654,16 @@ gst_rtmp_src_do_seek (GstBaseSrc * basesrc, GstSegment * segment) return FALSE; } -success: - /* This is set here so that the call to create() above doesn't clear it */ - src->discont = TRUE; - GST_DEBUG_OBJECT (src, "Seek to %" GST_TIME_FORMAT " successful", GST_TIME_ARGS (segment->start)); return TRUE; } -static gboolean -gst_rtmp_src_connect (GstRTMPSrc * src) -{ - RTMP_Init (src->rtmp); - src->rtmp->Link.timeout = src->timeout; - if (!RTMP_SetupURL (src->rtmp, src->uri)) { - GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, (NULL), - ("Failed to setup URL '%s'", src->uri)); - return FALSE; - } - src->seekable = !(src->rtmp->Link.lFlags & RTMP_LF_LIVE); - GST_INFO_OBJECT (src, "seekable %d", src->seekable); - - /* open if required */ - if (!RTMP_IsConnected (src->rtmp)) { - if (!RTMP_Connect (src->rtmp, NULL)) { - GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, (NULL), - ("Could not connect to RTMP stream \"%s\" for reading", src->uri)); - return FALSE; - } - } - - return TRUE; -} +#define STR2AVAL(av,str) G_STMT_START { \ + av.av_val = str; \ + av.av_len = strlen(av.av_val); \ +} G_STMT_END; /* open the file, do stuff necessary to go to PAUSED state */ static gboolean @@ -681,14 +682,24 @@ gst_rtmp_src_start (GstBaseSrc * basesrc) src->last_timestamp = 0; src->discont = TRUE; + src->first = TRUE; + src->connecting = FALSE; + src->rtmp = RTMP_Alloc (); if (!src->rtmp) { GST_ERROR_OBJECT (src, "Could not allocate librtmp's RTMP context"); goto error; } - if (!gst_rtmp_src_connect (src)) + RTMP_Init (src->rtmp); + src->rtmp->Link.timeout = src->timeout; + if (!RTMP_SetupURL (src->rtmp, src->uri)) { + GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, (NULL), + ("Failed to setup URL '%s'", src->uri)); goto error; + } + src->seekable = !(src->rtmp->Link.lFlags & RTMP_LF_LIVE); + GST_INFO_OBJECT (src, "seekable %d", src->seekable); return TRUE; @@ -700,18 +711,39 @@ gst_rtmp_src_start (GstBaseSrc * basesrc) return FALSE; } +#undef STR2AVAL + static gboolean gst_rtmp_src_unlock (GstBaseSrc * basesrc) { - GstRTMPSrc *rtmpsrc = GST_RTMP_SRC (basesrc); + GstRTMPSrc *src = GST_RTMP_SRC (basesrc); + + if (src->rtmp == NULL) + return TRUE; + + /* Check to see if we currently are doing any activity towards librtmp */ + GST_DEBUG_OBJECT (src, "Trying to lock"); - GST_DEBUG_OBJECT (rtmpsrc, "unlock"); + if (!RTMP_TRYLOCK (src)) { + GST_DEBUG_OBJECT (src, "Lock NOT aquired..."); + /* if we are trying to connect, but the internal socket are not yet + initialized, we keep trying until either connection have failed or + the socket comes up */ + while (src->connecting && src->rtmp->m_sb.sb_socket == -1) { + g_thread_yield (); + } + + if (src->rtmp->m_sb.sb_socket >= 0) { + GST_DEBUG_OBJECT (src, "Shutting down internal librtmp socket"); + shutdown (src->rtmp->m_sb.sb_socket, SHUT_RDWR); + } - /* This cancels the recv() underlying RTMP_Read, but will cause a - * SIGPIPE. Hopefully the app is ignoring it, or you've patched - * librtmp. */ - if (rtmpsrc->rtmp && rtmpsrc->rtmp->m_sb.sb_socket > 0) { - shutdown (rtmpsrc->rtmp->m_sb.sb_socket, SHUT_RDWR); + } else { + GST_DEBUG_OBJECT (src, "Lock aquired..."); + RTMP_Close (src->rtmp); + RTMP_Free (src->rtmp); + src->rtmp = NULL; + RTMP_UNLOCK (src); } return TRUE; diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.h b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.h index 60ebcb226a3..b4f7dae2643 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.h +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.h @@ -64,6 +64,10 @@ struct _GstRTMPSrc RTMP *rtmp; int timeout; + GMutex rtmp_lock; + gboolean connecting; + gboolean first; + gint64 cur_offset; GstClockTime last_timestamp; gboolean seekable; diff --git a/tests/check/elements/rtmp.c b/tests/check/elements/rtmp.c index 0615f1016e9..04315b5c41e 100644 --- a/tests/check/elements/rtmp.c +++ b/tests/check/elements/rtmp.c @@ -99,6 +99,29 @@ GST_START_TEST (rtmpsink_unlock_race) GST_END_TEST; +GST_START_TEST(rtmpsrc_unlock) +{ + gint fd; + GstHarness *h; + GTimer *timer; + + fd = add_listen_fd (23000); + h = gst_harness_new_parse ( + "rtmpsrc location=rtmp://localhost:23000/app/streamname1"); + + timer = g_timer_new (); + gst_harness_play (h); + + if (__i__ == 1) + g_usleep (G_USEC_PER_SEC / 10); + + gst_harness_teardown (h); + fail_unless (g_timer_elapsed (timer, NULL) < 2.0); + + g_timer_destroy (timer); + close (fd); +} +GST_END_TEST; static Suite * rtmp_suite (void) @@ -110,6 +133,8 @@ rtmp_suite (void) tcase_add_loop_test (tc_chain, rtmpsink_unlock, 0, 2); tcase_add_test (tc_chain, rtmpsink_unlock_race); + tcase_add_loop_test (tc_chain, rtmpsrc_unlock, 0, 2); + return s; } From ab67f054d45920bbceb9560a33d21a481a4f2c84 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 29 Apr 2016 11:14:04 +0200 Subject: [PATCH 075/339] rtmpsrc: rtmpsink: don't do close on internal sockets --- subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c index ac66f9bd7c0..fa491f739d2 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c @@ -241,7 +241,6 @@ gst_rtmp_sink_unlock (GstBaseSink * basesink) if (sink->rtmp->m_sb.sb_socket >= 0) { GST_DEBUG_OBJECT (sink, "Shutting down internal librtmp socket"); shutdown (sink->rtmp->m_sb.sb_socket, SHUT_RDWR); - close (sink->rtmp->m_sb.sb_socket); } } else { @@ -295,6 +294,7 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) if (sink->first) { /* open the connection */ + sink->first = FALSE; sink->connecting = TRUE; RTMP_LOCK (sink); @@ -320,7 +320,6 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) gst_buffer_ref (buf)); need_unref = TRUE; } - sink->first = FALSE; } GST_LOG_OBJECT (sink, "Sending %" G_GSIZE_FORMAT " bytes to RTMP server", From d258f921ba4b863fdb2d4ccd59bca4fe23c52dfc Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 29 Apr 2016 14:42:50 +0200 Subject: [PATCH 076/339] rtmpsink: return FLOW_OK always FLOW_ERROR seems to create havoc for us internally, so blaming it for some troubles with shutdown. --- subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c index fa491f739d2..ff0bcaa305e 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c @@ -348,7 +348,7 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) if (need_unref) gst_buffer_unref (buf); sink->have_write_error = TRUE; - return GST_FLOW_ERROR; + return GST_FLOW_OK; } connection_failed: From 81605b59796882b8657c2bbe8483aa441a619af7 Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Fri, 31 Jul 2015 10:24:51 +0200 Subject: [PATCH 077/339] h264parse: Support output of nalu-stream format Support converting to nalu-stream format - one NALU per buffer without startcodes. --- .../gst/videoparsers/gsth264parse.c | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c index 3b44beda4ee..fa2428f6e42 100644 --- a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c +++ b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c @@ -52,7 +52,8 @@ enum GST_H264_PARSE_FORMAT_NONE, GST_H264_PARSE_FORMAT_AVC, GST_H264_PARSE_FORMAT_BYTE, - GST_H264_PARSE_FORMAT_AVC3 + GST_H264_PARSE_FORMAT_AVC3, + GST_H264_PARSE_FORMAT_NALU }; enum @@ -95,8 +96,9 @@ static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS ("video/x-h264, parsed = (boolean) true, " "stream-format=(string) { avc, avc3, byte-stream }, " - "alignment=(string) { au, nal }")); - + "alignment=(string) { au, nal }; " + "video/x-h264, parsed = (boolean) true, " + "stream-format=(string) nalu-stream, alignment=(string) nal")); #define parent_class gst_h264_parse_parent_class G_DEFINE_TYPE (GstH264Parse, gst_h264_parse, GST_TYPE_BASE_PARSE); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (h264parse, "h264parse", @@ -362,6 +364,8 @@ gst_h264_parse_get_string (GstH264Parse * parse, gboolean format, gint code) return "byte-stream"; case GST_H264_PARSE_FORMAT_AVC3: return "avc3"; + case GST_H264_PARSE_FORMAT_NALU: + return "nalu-stream"; default: return "none"; } @@ -402,6 +406,8 @@ gst_h264_parse_format_from_caps (GstCaps * caps, guint * format, guint * align) *format = GST_H264_PARSE_FORMAT_BYTE; else if (strcmp (str, "avc3") == 0) *format = GST_H264_PARSE_FORMAT_AVC3; + else if (strcmp (str, "nalu-stream") == 0) + *format = GST_H264_PARSE_FORMAT_NALU; } } @@ -462,6 +468,8 @@ gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format, format = GST_H264_PARSE_FORMAT_BYTE; if (!align) align = GST_H264_PARSE_ALIGN_AU; + if (format == GST_H264_PARSE_FORMAT_NALU) + align = GST_H264_PARSE_ALIGN_NAL; GST_DEBUG_OBJECT (h264parse, "selected format %s, alignment %s", gst_h264_parse_get_string (h264parse, TRUE, format), @@ -470,8 +478,8 @@ gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format, h264parse->format = format; h264parse->align = align; - h264parse->transform = in_format != h264parse->format || - align == GST_H264_PARSE_ALIGN_AU; + h264parse->transform = (in_format != h264parse->format || + align == GST_H264_PARSE_ALIGN_AU) && h264parse->format != GST_H264_PARSE_FORMAT_NALU; if (caps) gst_caps_unref (caps); @@ -491,12 +499,14 @@ gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data, if (format == GST_H264_PARSE_FORMAT_AVC || format == GST_H264_PARSE_FORMAT_AVC3) { tmp = GUINT32_TO_BE (size << (32 - 8 * nl)); - } else { + } else if (format == GST_H264_PARSE_FORMAT_BYTE) { /* HACK: nl should always be 4 here, otherwise this won't work. * There are legit cases where nl in avc stream is 2, but byte-stream * SC is still always 4 bytes. */ nl = 4; tmp = GUINT32_TO_BE (1); + } else { + nl = 0; } gst_buffer_fill (buf, 0, &tmp, sizeof (guint32)); @@ -1292,6 +1302,12 @@ gst_h264_parse_handle_frame_packetized (GstBaseParse * parse, * Real data is either taken from input by baseclass or * a replacement output buffer is provided anyway. */ gst_h264_parse_parse_frame (parse, &tmp_frame); + + if (h264parse->format == GST_H264_PARSE_FORMAT_NALU) { + g_assert (tmp_frame.out_buffer == NULL); + tmp_frame.out_buffer = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL, nalu.offset, nalu.size); + gst_buffer_copy_into (tmp_frame.out_buffer, tmp_frame.buffer, GST_BUFFER_COPY_METADATA, 0, -1); + } ret = gst_base_parse_finish_frame (parse, &tmp_frame, nl + nalu.size); left -= nl + nalu.size; } @@ -1585,6 +1601,12 @@ gst_h264_parse_handle_frame (GstBaseParse * parse, gst_h264_parse_parse_frame (parse, frame); + if (h264parse->format == GST_H264_PARSE_FORMAT_NALU) { + g_assert (frame->out_buffer == NULL); + frame->out_buffer = gst_buffer_copy_region (frame->buffer, GST_BUFFER_COPY_ALL, nalu.offset, nalu.size); + gst_buffer_copy_into (frame->out_buffer, frame->buffer, GST_BUFFER_COPY_METADATA, 0, -1); + } + return gst_base_parse_finish_frame (parse, frame, framesize); more: @@ -3584,7 +3606,7 @@ gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps) } /* packetized video has codec_data (required for AVC, optional for AVC3) */ - if (codec_data_value != NULL) { + if (format != GST_H264_PARSE_FORMAT_NALU && codec_data_value != NULL) { GstMapInfo map; guint i; @@ -3637,7 +3659,7 @@ gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps) /* don't confuse codec_data with inband sps/pps */ h264parse->have_sps_in_frame = FALSE; h264parse->have_pps_in_frame = FALSE; - } else if (format == GST_H264_PARSE_FORMAT_BYTE) { + } else if (format == GST_H264_PARSE_FORMAT_BYTE || format == GST_H264_PARSE_FORMAT_NALU) { GST_DEBUG_OBJECT (h264parse, "have bytestream h264"); /* nothing to pre-process */ h264parse->packetized = FALSE; From 22c1fc5d75f251689e2be2a204fd3b37e8ab4467 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 15 Jun 2017 11:05:48 +0200 Subject: [PATCH 078/339] pcapparse: add stats-property and refactor It is very useful to be able to extract information about the available streams in the pcap, for the purpose of using filtering to access one or more of them. Other changes: * keep ip-addresses as dynamic strings to avoid const problems with calling get_ip_address_as_string twice, and comparisons. * have timestamps start from 0 * Move _GstPcapParse struct in as "private" * Move _init and _class_init etc to the bottom to avoid forward declarations --- .../gst/pcapparse/gstpcapparse.c | 628 +++++++++++------- .../gst/pcapparse/gstpcapparse.h | 50 -- 2 files changed, 381 insertions(+), 297 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c index 53e7ef6fc25..9615da20966 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c @@ -46,6 +46,8 @@ #include #endif +#define GLIB_DISABLE_DEPRECATION_WARNINGS + #include "gstpcapparse.h" #include @@ -73,9 +75,56 @@ enum PROP_SRC_PORT, PROP_DST_PORT, PROP_CAPS, - PROP_TS_OFFSET + PROP_TS_OFFSET, + PROP_STATS +}; + +typedef enum +{ + PCAP_PARSE_STATE_CREATED, + PCAP_PARSE_STATE_PARSING, +} GstPcapParseState; + +typedef enum +{ + LINKTYPE_ETHER = 1, + LINKTYPE_RAW = 101, + LINKTYPE_SLL = 113, + LINKTYPE_SLL2 = 276 +} GstPcapParseLinktype; + +struct _GstPcapParse +{ + GstElement element; + + /*< private > */ + GstPad *sink_pad; + GstPad *src_pad; + + /* properties */ + gchar *src_ip; + gchar *dst_ip; + gint src_port; + gint dst_port; + GstCaps *caps; + gint64 offset; + + /* state */ + GstAdapter * adapter; + GHashTable *stats_map; + gboolean initialized; + gboolean swap_endian; + gboolean nanosecond_timestamp; + gint64 cur_packet_size; + GstClockTime cur_ts; + GstClockTime base_ts; + GstPcapParseLinktype linktype; + + gboolean newsegment_sent; + gboolean first_packet; }; + GST_DEBUG_CATEGORY_STATIC (gst_pcap_parse_debug); #define GST_CAT_DEFAULT gst_pcap_parse_debug @@ -89,231 +138,25 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); -static void gst_pcap_parse_finalize (GObject * object); -static void gst_pcap_parse_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec); -static void gst_pcap_parse_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec); -static GstStateChangeReturn -gst_pcap_parse_change_state (GstElement * element, GstStateChange transition); - -static void gst_pcap_parse_reset (GstPcapParse * self); - -static GstFlowReturn gst_pcap_parse_chain (GstPad * pad, - GstObject * parent, GstBuffer * buffer); -static gboolean gst_pcap_sink_event (GstPad * pad, - GstObject * parent, GstEvent * event); - - #define parent_class gst_pcap_parse_parent_class G_DEFINE_TYPE (GstPcapParse, gst_pcap_parse, GST_TYPE_ELEMENT); GST_ELEMENT_REGISTER_DEFINE (pcapparse, "pcapparse", GST_RANK_NONE, GST_TYPE_PCAP_PARSE); -static void -gst_pcap_parse_class_init (GstPcapParseClass * klass) -{ - GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstElementClass *element_class = GST_ELEMENT_CLASS (klass); - - gobject_class->finalize = gst_pcap_parse_finalize; - gobject_class->get_property = gst_pcap_parse_get_property; - gobject_class->set_property = gst_pcap_parse_set_property; - - g_object_class_install_property (gobject_class, - PROP_SRC_IP, g_param_spec_string ("src-ip", "Source IP", - "Source IP to restrict to", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, - PROP_DST_IP, g_param_spec_string ("dst-ip", "Destination IP", - "Destination IP to restrict to", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, - PROP_SRC_PORT, g_param_spec_int ("src-port", "Source port", - "Source port to restrict to", -1, G_MAXUINT16, -1, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, - PROP_DST_PORT, g_param_spec_int ("dst-port", "Destination port", - "Destination port to restrict to", -1, G_MAXUINT16, -1, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, PROP_CAPS, - g_param_spec_boxed ("caps", "Caps", - "The caps of the source pad", GST_TYPE_CAPS, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, PROP_TS_OFFSET, - g_param_spec_int64 ("ts-offset", "Timestamp Offset", - "Relative timestamp offset (ns) to apply (-1 = use absolute packet time)", - -1, G_MAXINT64, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - gst_element_class_add_static_pad_template (element_class, &sink_template); - gst_element_class_add_static_pad_template (element_class, &src_template); - - element_class->change_state = gst_pcap_parse_change_state; - - gst_element_class_set_static_metadata (element_class, "PCapParse", - "Raw/Parser", - "Parses a raw pcap stream", - "Ole André Vadla Ravnås "); - - GST_DEBUG_CATEGORY_INIT (gst_pcap_parse_debug, "pcapparse", 0, "pcap parser"); -} - -static void -gst_pcap_parse_init (GstPcapParse * self) -{ - self->sink_pad = gst_pad_new_from_static_template (&sink_template, "sink"); - gst_pad_set_chain_function (self->sink_pad, - GST_DEBUG_FUNCPTR (gst_pcap_parse_chain)); - gst_pad_use_fixed_caps (self->sink_pad); - gst_pad_set_event_function (self->sink_pad, - GST_DEBUG_FUNCPTR (gst_pcap_sink_event)); - gst_element_add_pad (GST_ELEMENT (self), self->sink_pad); - - self->src_pad = gst_pad_new_from_static_template (&src_template, "src"); - gst_pad_use_fixed_caps (self->src_pad); - gst_element_add_pad (GST_ELEMENT (self), self->src_pad); - - self->src_ip = -1; - self->dst_ip = -1; - self->src_port = -1; - self->dst_port = -1; - self->offset = -1; - - self->adapter = gst_adapter_new (); - - gst_pcap_parse_reset (self); -} - -static void -gst_pcap_parse_finalize (GObject * object) -{ - GstPcapParse *self = GST_PCAP_PARSE (object); - - g_object_unref (self->adapter); - if (self->caps) - gst_caps_unref (self->caps); - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - -static const gchar * -get_ip_address_as_string (gint64 ip_addr) -{ - if (ip_addr >= 0) { - struct in_addr addr; - addr.s_addr = ip_addr; - return inet_ntoa (addr); - } else { - return ""; - } -} - -static void -set_ip_address_from_string (gint64 * ip_addr, const gchar * ip_str) -{ - if (ip_str[0] != '\0') { - gulong addr = inet_addr (ip_str); - if (addr != INADDR_NONE) - *ip_addr = addr; - } else { - *ip_addr = -1; - } -} - -static void -gst_pcap_parse_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstPcapParse *self = GST_PCAP_PARSE (object); - - switch (prop_id) { - case PROP_SRC_IP: - g_value_set_string (value, get_ip_address_as_string (self->src_ip)); - break; - - case PROP_DST_IP: - g_value_set_string (value, get_ip_address_as_string (self->dst_ip)); - break; - - case PROP_SRC_PORT: - g_value_set_int (value, self->src_port); - break; - - case PROP_DST_PORT: - g_value_set_int (value, self->dst_port); - break; - - case PROP_CAPS: - gst_value_set_caps (value, self->caps); - break; - - case PROP_TS_OFFSET: - g_value_set_int64 (value, self->offset); - break; +#define ETH_HEADER_LEN 14 +#define SLL_HEADER_LEN 16 +#define IP_HEADER_MIN_LEN 20 +#define UDP_HEADER_LEN 8 - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} +#define IP_PROTO_UDP 17 +#define IP_PROTO_TCP 6 -static void -gst_pcap_parse_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) +static gchar * +get_ip_address_as_string (guint32 ip_addr) { - GstPcapParse *self = GST_PCAP_PARSE (object); - - switch (prop_id) { - case PROP_SRC_IP: - set_ip_address_from_string (&self->src_ip, g_value_get_string (value)); - break; - - case PROP_DST_IP: - set_ip_address_from_string (&self->dst_ip, g_value_get_string (value)); - break; - - case PROP_SRC_PORT: - self->src_port = g_value_get_int (value); - break; - - case PROP_DST_PORT: - self->dst_port = g_value_get_int (value); - break; - - case PROP_CAPS: - { - const GstCaps *new_caps_val; - GstCaps *new_caps, *old_caps; - - new_caps_val = gst_value_get_caps (value); - if (new_caps_val == NULL) { - new_caps = gst_caps_new_any (); - } else { - new_caps = gst_caps_copy (new_caps_val); - } - - old_caps = self->caps; - self->caps = new_caps; - if (old_caps) - gst_caps_unref (old_caps); - - gst_pad_set_caps (self->src_pad, new_caps); - break; - } - - case PROP_TS_OFFSET: - self->offset = g_value_get_int64 (value); - break; - - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } + struct in_addr addr; + addr.s_addr = ip_addr; + return g_strdup (inet_ntoa (addr)); } static void @@ -329,6 +172,7 @@ gst_pcap_parse_reset (GstPcapParse * self) self->first_packet = TRUE; gst_adapter_clear (self->adapter); + g_hash_table_remove_all (self->stats_map); } static guint32 @@ -355,15 +199,78 @@ gst_pcap_parse_read_uint32 (GstPcapParse * self, const guint8 * p) #define IP_HEADER_MIN_LEN 20 #define UDP_HEADER_LEN 8 -#define IP_PROTO_UDP 17 -#define IP_PROTO_TCP 6 +static GValueArray * +gst_pcap_parse_get_stats (GstPcapParse * self) +{ + GValueArray *ret; + GList *streams, *walk; + guint len, i; + + len = g_hash_table_size (self->stats_map); + ret = g_value_array_new (len); + + walk = streams = g_hash_table_get_values (self->stats_map); + for (i = 0; i < len; i++) { + GstStructure *s = walk->data; + GValue *value; + ret->n_values++; + value = g_value_array_get_nth (ret, i); + GST_INFO_OBJECT (self, "Adding stats %d: %" GST_PTR_FORMAT, i, s); + + g_value_init (value, GST_TYPE_STRUCTURE); + gst_value_set_structure (value, s); + + walk = walk->next; + } + + g_list_free (streams); + + return ret; +} + +static void +gst_pcap_parse_add_stats (GstPcapParse * self, + const gchar * src_ip, guint16 src_port, + const gchar * dst_ip, guint16 dst_port, gint payload_size) +{ + GstStructure *s; + gint packets; + gint bytes; + + gchar *key_str = g_strdup_printf ("%s:%d->%s:%d", + src_ip, src_port, dst_ip, dst_port); + + s = g_hash_table_lookup (self->stats_map, key_str); + if (s == NULL) { + s = gst_structure_new ("stats", + "first-ts", G_TYPE_UINT64, self->cur_ts, + "id-str", G_TYPE_STRING, key_str, + "src-ip", G_TYPE_STRING, src_ip, + "src-port", G_TYPE_INT, src_port, + "dst-ip", G_TYPE_STRING, dst_ip, + "dst-port", G_TYPE_INT, dst_port, + "packets", G_TYPE_INT, 0, "bytes", G_TYPE_INT, 0, NULL); + g_hash_table_insert (self->stats_map, g_strdup (key_str), s); + } + g_free (key_str); + + gst_structure_get (s, + "packets", G_TYPE_INT, &packets, "bytes", G_TYPE_INT, &bytes, NULL); + + packets += 1; + bytes += payload_size; + + gst_structure_set (s, + "packets", G_TYPE_INT, packets, "bytes", G_TYPE_INT, bytes, NULL); +} static gboolean gst_pcap_parse_scan_frame (GstPcapParse * self, const guint8 * buf, gint buf_size, const guint8 ** payload, gint * payload_size) { + gboolean ret = FALSE; const guint8 *buf_ip = 0; const guint8 *buf_proto; guint16 eth_type; @@ -374,6 +281,8 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, guint8 ip_protocol; guint32 ip_src_addr; guint32 ip_dst_addr; + gchar *src_ip = NULL; + gchar *dst_ip = NULL; guint16 src_port; guint16 dst_port; guint16 len; @@ -382,14 +291,14 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, switch (self->linktype) { case LINKTYPE_ETHER: if (buf_size < ETH_HEADER_LEN + IP_HEADER_MIN_LEN + UDP_HEADER_LEN) - return FALSE; + goto done; eth_type = GUINT16_FROM_BE (*((guint16 *) (buf + ETH_MAC_ADDRESSES_LEN))); /* check for vlan 802.1q header (4 bytes, with first two bytes equal to 0x8100) */ if (eth_type == 0x8100) { if (buf_size < ETH_HEADER_LEN + ETH_VLAN_HEADER_LEN + IP_HEADER_MIN_LEN + UDP_HEADER_LEN) - return FALSE; + goto done; eth_type = GUINT16_FROM_BE (*((guint16 *) (buf + ETH_MAC_ADDRESSES_LEN + ETH_VLAN_HEADER_LEN))); @@ -400,7 +309,7 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, break; case LINKTYPE_SLL: if (buf_size < SLL_HEADER_LEN + IP_HEADER_MIN_LEN + UDP_HEADER_LEN) - return FALSE; + goto done; eth_type = GUINT16_FROM_BE (*((guint16 *) (buf + 14))); buf_ip = buf + SLL_HEADER_LEN; @@ -414,32 +323,32 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, break; case LINKTYPE_RAW: if (buf_size < IP_HEADER_MIN_LEN + UDP_HEADER_LEN) - return FALSE; + goto done; eth_type = 0x800; /* This is fine since IPv4/IPv6 is parse elsewhere */ buf_ip = buf; break; default: - return FALSE; + goto done; } if (eth_type != 0x800) { GST_ERROR_OBJECT (self, "Link type %d: Ethernet type %d is not supported; only type 0x800", (gint) self->linktype, (gint) eth_type); - return FALSE; + goto done; } b = *buf_ip; /* Check that the packet is IPv4 */ if (((b >> 4) & 0x0f) != 4) - return FALSE; + goto done; ip_header_size = (b & 0x0f) * 4; if (buf_ip + ip_header_size > buf + buf_size) - return FALSE; + goto done; flags = buf_ip[6] >> 5; fragment_offset = @@ -453,7 +362,7 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, GST_LOG_OBJECT (self, "ip proto %d", (gint) ip_protocol); if (ip_protocol != IP_PROTO_UDP && ip_protocol != IP_PROTO_TCP) - return FALSE; + goto done; /* ip info */ ip_src_addr = *((guint32 *) (buf_ip + 12)); @@ -469,36 +378,60 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, if (ip_protocol == IP_PROTO_UDP) { len = GUINT16_FROM_BE (*((guint16 *) (buf_proto + 4))); if (len < UDP_HEADER_LEN || buf_proto + len > buf + buf_size) - return FALSE; + goto done; *payload = buf_proto + UDP_HEADER_LEN; *payload_size = len - UDP_HEADER_LEN; } else { if (buf_proto + 12 >= buf + buf_size) - return FALSE; + goto done; len = (buf_proto[12] >> 4) * 4; if (buf_proto + len > buf + buf_size) - return FALSE; + goto done; /* all remaining data following tcp header is payload */ *payload = buf_proto + len; *payload_size = ip_packet_len - ip_header_size - len; } + src_ip = get_ip_address_as_string (ip_src_addr); + dst_ip = get_ip_address_as_string (ip_dst_addr); + + gst_pcap_parse_add_stats (self, src_ip, src_port, dst_ip, dst_port, + *payload_size); + /* but still filter as configured */ - if (self->src_ip >= 0 && ip_src_addr != self->src_ip) - return FALSE; + if (self->src_ip && !g_str_equal (src_ip, self->src_ip)) { + GST_LOG_OBJECT (self, "Filtering on src-ip (%s != %s)", + src_ip, self->src_ip); + goto done; + } - if (self->dst_ip >= 0 && ip_dst_addr != self->dst_ip) - return FALSE; + if (self->dst_ip && !g_str_equal (dst_ip, self->dst_ip)) { + GST_LOG_OBJECT (self, "Filtering on dst-ip (%s != %s)", + dst_ip, self->dst_ip); + goto done; + } - if (self->src_port >= 0 && src_port != self->src_port) - return FALSE; + if (self->src_port >= 0 && src_port != self->src_port) { + GST_LOG_OBJECT (self, "Filtering on src-port (%d != %d)", + src_port, self->src_port); + goto done; + } - if (self->dst_port >= 0 && dst_port != self->dst_port) - return FALSE; + if (self->dst_port >= 0 && dst_port != self->dst_port) { + GST_LOG_OBJECT (self, "Filtering on dst-port (%d != %d)", + dst_port, self->dst_port); + goto done; + } + + ret = TRUE; - return TRUE; +done: + g_free (src_ip); + g_free (dst_ip); + + return ret; } static GstFlowReturn @@ -560,10 +493,12 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) self->cur_packet_size - offset - payload_size); if (GST_CLOCK_TIME_IS_VALID (self->cur_ts)) { - if (!GST_CLOCK_TIME_IS_VALID (self->base_ts)) + if (!GST_CLOCK_TIME_IS_VALID (self->base_ts)) { self->base_ts = self->cur_ts; + GST_DEBUG_OBJECT (self, "Setting base_ts to %" GST_TIME_FORMAT, + GST_TIME_ARGS (self->base_ts)); + } if (self->offset >= 0) { - self->cur_ts -= self->base_ts; self->cur_ts += self->offset; } } @@ -666,13 +601,13 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) } if (list) { - if (!self->newsegment_sent && GST_CLOCK_TIME_IS_VALID (self->cur_ts)) { + if (!self->newsegment_sent && GST_CLOCK_TIME_IS_VALID (self->base_ts)) { GstSegment segment; if (self->caps) gst_pad_set_caps (self->src_pad, self->caps); gst_segment_init (&segment, GST_FORMAT_TIME); - segment.start = self->base_ts; + gst_segment_set_running_time (&segment, GST_FORMAT_TIME, self->base_ts); gst_pad_push_event (self->src_pad, gst_event_new_segment (&segment)); self->newsegment_sent = TRUE; } @@ -689,6 +624,103 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) return ret; } +static void +gst_pcap_parse_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstPcapParse *self = GST_PCAP_PARSE (object); + + switch (prop_id) { + case PROP_SRC_IP: + g_value_set_string (value, self->src_ip); + break; + + case PROP_DST_IP: + g_value_set_string (value, self->dst_ip); + break; + + case PROP_SRC_PORT: + g_value_set_int (value, self->src_port); + break; + + case PROP_DST_PORT: + g_value_set_int (value, self->dst_port); + break; + + case PROP_CAPS: + gst_value_set_caps (value, self->caps); + break; + + case PROP_TS_OFFSET: + g_value_set_int64 (value, self->offset); + break; + + case PROP_STATS: + g_value_take_boxed (value, gst_pcap_parse_get_stats (self)); + break; + + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_pcap_parse_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstPcapParse *self = GST_PCAP_PARSE (object); + + switch (prop_id) { + case PROP_SRC_IP: + g_free (self->src_ip); + self->src_ip = g_strdup (g_value_get_string (value)); + break; + + case PROP_DST_IP: + g_free (self->dst_ip); + self->dst_ip = g_strdup (g_value_get_string (value)); + break; + + case PROP_SRC_PORT: + self->src_port = g_value_get_int (value); + break; + + case PROP_DST_PORT: + self->dst_port = g_value_get_int (value); + break; + + case PROP_CAPS: + { + const GstCaps *new_caps_val; + GstCaps *new_caps, *old_caps; + + new_caps_val = gst_value_get_caps (value); + if (new_caps_val == NULL) { + new_caps = gst_caps_new_any (); + } else { + new_caps = gst_caps_copy (new_caps_val); + } + + old_caps = self->caps; + self->caps = new_caps; + if (old_caps) + gst_caps_unref (old_caps); + + gst_pad_set_caps (self->src_pad, new_caps); + break; + } + + case PROP_TS_OFFSET: + self->offset = g_value_get_int64 (value); + break; + + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + static gboolean gst_pcap_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -721,13 +753,115 @@ gst_pcap_parse_change_state (GstElement * element, GstStateChange transition) ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); switch (transition) { - case GST_STATE_CHANGE_PAUSED_TO_READY: + case GST_STATE_CHANGE_NULL_TO_READY: gst_pcap_parse_reset (self); break; default: break; } - return ret; } + +static void +gst_pcap_parse_finalize (GObject * object) +{ + GstPcapParse *self = GST_PCAP_PARSE (object); + + g_object_unref (self->adapter); + + /* to get a stats-summary in the debug-log */ + g_value_array_free (gst_pcap_parse_get_stats (self)); + g_hash_table_destroy (self->stats_map); + + g_free (self->src_ip); + g_free (self->dst_ip); + + if (self->caps) + gst_caps_unref (self->caps); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_pcap_parse_class_init (GstPcapParseClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + + gobject_class->finalize = gst_pcap_parse_finalize; + gobject_class->get_property = gst_pcap_parse_get_property; + gobject_class->set_property = gst_pcap_parse_set_property; + + g_object_class_install_property (gobject_class, + PROP_SRC_IP, g_param_spec_string ("src-ip", "Source IP", + "Source IP to restrict to", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_DST_IP, g_param_spec_string ("dst-ip", "Destination IP", + "Destination IP to restrict to", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_SRC_PORT, g_param_spec_int ("src-port", "Source port", + "Source port to restrict to", -1, G_MAXUINT16, -1, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_DST_PORT, g_param_spec_int ("dst-port", "Destination port", + "Destination port to restrict to", -1, G_MAXUINT16, -1, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_CAPS, + g_param_spec_boxed ("caps", "Caps", + "The caps of the source pad", GST_TYPE_CAPS, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_TS_OFFSET, + g_param_spec_int64 ("ts-offset", "Timestamp Offset", + "Relative timestamp offset (ns) to apply (-1 = use absolute packet time)", + -1, G_MAXINT64, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_STATS, + g_param_spec_boxed ("stats", "Stats", + "Some stats for the different streams parsed", G_TYPE_VALUE_ARRAY, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + gst_element_class_add_static_pad_template (element_class, &sink_template); + gst_element_class_add_static_pad_template (element_class, &src_template); + + element_class->change_state = gst_pcap_parse_change_state; + + gst_element_class_set_static_metadata (element_class, "PCapParse", + "Raw/Parser", + "Parses a raw pcap stream", + "Ole André Vadla Ravnås "); + + GST_DEBUG_CATEGORY_INIT (gst_pcap_parse_debug, "pcapparse", 0, "pcap parser"); +} + +static void +gst_pcap_parse_init (GstPcapParse * self) +{ + self->sink_pad = gst_pad_new_from_static_template (&sink_template, "sink"); + gst_pad_set_chain_function (self->sink_pad, + GST_DEBUG_FUNCPTR (gst_pcap_parse_chain)); + gst_pad_use_fixed_caps (self->sink_pad); + gst_pad_set_event_function (self->sink_pad, + GST_DEBUG_FUNCPTR (gst_pcap_sink_event)); + gst_element_add_pad (GST_ELEMENT (self), self->sink_pad); + + self->src_pad = gst_pad_new_from_static_template (&src_template, "src"); + gst_pad_use_fixed_caps (self->src_pad); + gst_element_add_pad (GST_ELEMENT (self), self->src_pad); + + self->src_port = -1; + self->dst_port = -1; + self->offset = -1; + + self->adapter = gst_adapter_new (); + self->stats_map = + g_hash_table_new_full (g_str_hash, g_str_equal, g_free, + (GDestroyNotify) gst_structure_free); +} diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.h b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.h index 9547ce4003e..7e866bcfaf3 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.h +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.h @@ -39,56 +39,6 @@ G_BEGIN_DECLS typedef struct _GstPcapParse GstPcapParse; typedef struct _GstPcapParseClass GstPcapParseClass; -typedef enum -{ - PCAP_PARSE_STATE_CREATED, - PCAP_PARSE_STATE_PARSING, -} GstPcapParseState; - -typedef enum -{ - LINKTYPE_ETHER = 1, - LINKTYPE_RAW = 101, - LINKTYPE_SLL = 113, - LINKTYPE_SLL2 = 276 -} GstPcapParseLinktype; - -/** - * GstPcapParse: - * - * GstPcapParse element. - */ - -struct _GstPcapParse -{ - GstElement element; - - /*< private >*/ - GstPad * sink_pad; - GstPad * src_pad; - - /* properties */ - gint64 src_ip; - gint64 dst_ip; - gint32 src_port; - gint32 dst_port; - GstCaps *caps; - gint64 offset; - - /* state */ - GstAdapter * adapter; - gboolean initialized; - gboolean swap_endian; - gboolean nanosecond_timestamp; - gint64 cur_packet_size; - GstClockTime cur_ts; - GstClockTime base_ts; - GstPcapParseLinktype linktype; - - gboolean newsegment_sent; - gboolean first_packet; -}; - struct _GstPcapParseClass { GstElementClass parent_class; From da30aaec72ff10b21f1f6155c80c6c33557b3842 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 16 Jun 2017 09:44:56 +0200 Subject: [PATCH 079/339] pcapparse: try to detect presence of RTP and RTCP in the stats --- .../gst/pcapparse/gstpcapparse.c | 89 ++++++++++++++++++- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c index 9615da20966..1ae4232162f 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c @@ -229,10 +229,91 @@ gst_pcap_parse_get_stats (GstPcapParse * self) return ret; } +/* from gstrtpbuffer.c */ +typedef struct _GstRTPHeader +{ +#if G_BYTE_ORDER == G_LITTLE_ENDIAN + unsigned int csrc_count:4; /* CSRC count */ + unsigned int extension:1; /* header extension flag */ + unsigned int padding:1; /* padding flag */ + unsigned int version:2; /* protocol version */ + unsigned int payload_type:7; /* payload type */ + unsigned int marker:1; /* marker bit */ +#elif G_BYTE_ORDER == G_BIG_ENDIAN + unsigned int version:2; /* protocol version */ + unsigned int padding:1; /* padding flag */ + unsigned int extension:1; /* header extension flag */ + unsigned int csrc_count:4; /* CSRC count */ + unsigned int marker:1; /* marker bit */ + unsigned int payload_type:7; /* payload type */ +#else +#error "G_BYTE_ORDER should be big or little endian." +#endif + unsigned int seq:16; /* sequence number */ + unsigned int timestamp:32; /* timestamp */ + unsigned int ssrc:32; /* synchronization source */ + guint8 csrclist[4]; /* optional CSRC list, 32 bits each */ +} GstRTPHeader; + +static void +_check_rtp_rtcp (const guint8 * payload, gint payload_size, + gboolean * is_rtp, gboolean * is_rtcp, gint * payload_type, guint32 * ssrc) +{ + GstRTPHeader *rtp; + *is_rtp = FALSE; + *is_rtcp = FALSE; + + /* minimum rtp-header length */ + if (payload_size < 12) + return; + + rtp = (GstRTPHeader *) payload; + /* check version (common for both RTP & RTCP) */ + if (rtp->version != 2) + return; + + /* for payload_type range 66-95 we assume RTCP, not RTP */ + if (rtp->payload_type >= 66 && rtp->payload_type <= 95) { + *is_rtcp = TRUE; + *ssrc = rtp->timestamp; /* Hackish, but true */ + } else { + *payload_type = rtp->payload_type; + *ssrc = rtp->ssrc; + *is_rtp = TRUE; + } +} + +static void +_add_rtp_stats (GstStructure * s, const guint8 * payload, gint payload_size) +{ + gboolean is_rtp; + gboolean is_rtcp; + gint payload_type; + guint32 ssrc; + + _check_rtp_rtcp (payload, payload_size, &is_rtp, &is_rtcp, &payload_type, + &ssrc); + + if (is_rtcp) { + gst_structure_set (s, "has-rtcp", G_TYPE_BOOLEAN, TRUE, NULL); + + gst_structure_set (s, "ssrc", G_TYPE_UINT, ssrc, NULL); + + } else if (is_rtp) { + gst_structure_set (s, "has-rtp", G_TYPE_BOOLEAN, TRUE, NULL); + + /* FIXME: support multiple pt/ssrc */ + gst_structure_set (s, + "payload-type", G_TYPE_INT, payload_type, + "ssrc", G_TYPE_UINT, ssrc, NULL); + } +} + static void gst_pcap_parse_add_stats (GstPcapParse * self, + const guint8 * payload, gint payload_size, const gchar * src_ip, guint16 src_port, - const gchar * dst_ip, guint16 dst_port, gint payload_size) + const gchar * dst_ip, guint16 dst_port) { GstStructure *s; gint packets; @@ -263,6 +344,8 @@ gst_pcap_parse_add_stats (GstPcapParse * self, gst_structure_set (s, "packets", G_TYPE_INT, packets, "bytes", G_TYPE_INT, bytes, NULL); + + _add_rtp_stats (s, payload, payload_size); } static gboolean @@ -397,8 +480,8 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, src_ip = get_ip_address_as_string (ip_src_addr); dst_ip = get_ip_address_as_string (ip_dst_addr); - gst_pcap_parse_add_stats (self, src_ip, src_port, dst_ip, dst_port, - *payload_size); + gst_pcap_parse_add_stats (self, *payload, *payload_size, + src_ip, src_port, dst_ip, dst_port); /* but still filter as configured */ if (self->src_ip && !g_str_equal (src_ip, self->src_ip)) { From 99d919b4b10f818db3e0db6376e1b8659571e939 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 19 Jun 2017 14:52:23 +0200 Subject: [PATCH 080/339] pcapparse: add configurable start-time Allows multiple parsings of the same pcap to be synchronized. (Like audio and video) --- .../gst/pcapparse/gstpcapparse.c | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c index 1ae4232162f..64c7dc8a720 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c @@ -76,7 +76,8 @@ enum PROP_DST_PORT, PROP_CAPS, PROP_TS_OFFSET, - PROP_STATS + PROP_STATS, + PROP_START_TIME, }; typedef enum @@ -108,6 +109,7 @@ struct _GstPcapParse gint dst_port; GstCaps *caps; gint64 offset; + GstClockTime start_ts; /* state */ GstAdapter * adapter; @@ -117,7 +119,6 @@ struct _GstPcapParse gboolean nanosecond_timestamp; gint64 cur_packet_size; GstClockTime cur_ts; - GstClockTime base_ts; GstPcapParseLinktype linktype; gboolean newsegment_sent; @@ -167,7 +168,7 @@ gst_pcap_parse_reset (GstPcapParse * self) self->nanosecond_timestamp = FALSE; self->cur_packet_size = -1; self->cur_ts = GST_CLOCK_TIME_NONE; - self->base_ts = GST_CLOCK_TIME_NONE; + self->start_ts = GST_CLOCK_TIME_NONE; self->newsegment_sent = FALSE; self->first_packet = TRUE; @@ -576,10 +577,10 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) self->cur_packet_size - offset - payload_size); if (GST_CLOCK_TIME_IS_VALID (self->cur_ts)) { - if (!GST_CLOCK_TIME_IS_VALID (self->base_ts)) { - self->base_ts = self->cur_ts; - GST_DEBUG_OBJECT (self, "Setting base_ts to %" GST_TIME_FORMAT, - GST_TIME_ARGS (self->base_ts)); + if (!GST_CLOCK_TIME_IS_VALID (self->start_ts)) { + self->start_ts = self->cur_ts; + GST_DEBUG_OBJECT (self, "Setting start_ts to %" GST_TIME_FORMAT, + GST_TIME_ARGS (self->start_ts)); } if (self->offset >= 0) { self->cur_ts += self->offset; @@ -684,13 +685,13 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) } if (list) { - if (!self->newsegment_sent && GST_CLOCK_TIME_IS_VALID (self->base_ts)) { + if (!self->newsegment_sent && GST_CLOCK_TIME_IS_VALID (self->start_ts)) { GstSegment segment; if (self->caps) gst_pad_set_caps (self->src_pad, self->caps); gst_segment_init (&segment, GST_FORMAT_TIME); - gst_segment_set_running_time (&segment, GST_FORMAT_TIME, self->base_ts); + gst_segment_set_running_time (&segment, GST_FORMAT_TIME, self->start_ts); gst_pad_push_event (self->src_pad, gst_event_new_segment (&segment)); self->newsegment_sent = TRUE; } @@ -738,6 +739,10 @@ gst_pcap_parse_get_property (GObject * object, guint prop_id, g_value_set_int64 (value, self->offset); break; + case PROP_START_TIME: + g_value_set_int64 (value, self->start_ts); + break; + case PROP_STATS: g_value_take_boxed (value, gst_pcap_parse_get_stats (self)); break; @@ -798,6 +803,10 @@ gst_pcap_parse_set_property (GObject * object, guint prop_id, self->offset = g_value_get_int64 (value); break; + case PROP_START_TIME: + self->start_ts = g_value_get_int64 (value); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -906,6 +915,11 @@ gst_pcap_parse_class_init (GstPcapParseClass * klass) "Relative timestamp offset (ns) to apply (-1 = use absolute packet time)", -1, G_MAXINT64, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_START_TIME, + g_param_spec_int64 ("start-time", "Start Time", + "The start-time (ns) in the segment (-1 = use first timestamp)", + -1, G_MAXINT64, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_STATS, g_param_spec_boxed ("stats", "Stats", "Some stats for the different streams parsed", G_TYPE_VALUE_ARRAY, From 2cf41891724863c868a0bedd015e30d86363b5b9 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 28 Jan 2019 17:19:16 +0100 Subject: [PATCH 081/339] pcapparse: Take control of the discont flag, don't copy from input Take control of the discont flag and set it only on the first buffer. Don't copy from the incoming buffer since that may set the discont flag for many consecutive packets if the incoming buffer contains multiple packets. --- .../gst-plugins-bad/gst/pcapparse/gstpcapparse.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c index 64c7dc8a720..29abb145760 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c @@ -123,6 +123,7 @@ struct _GstPcapParse gboolean newsegment_sent; gboolean first_packet; + gboolean discont; }; @@ -171,6 +172,7 @@ gst_pcap_parse_reset (GstPcapParse * self) self->start_ts = GST_CLOCK_TIME_NONE; self->newsegment_sent = FALSE; self->first_packet = TRUE; + self->discont = TRUE; gst_adapter_clear (self->adapter); g_hash_table_remove_all (self->stats_map); @@ -591,6 +593,15 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) if (list == NULL) list = gst_buffer_list_new (); + + /* Take control of the discont flag in order to not have one + * discont buffer be split into multiple */ + if (self->discont) + GST_BUFFER_FLAG_SET (out_buf, GST_BUFFER_FLAG_DISCONT); + else + GST_BUFFER_FLAG_UNSET (out_buf, GST_BUFFER_FLAG_DISCONT); + self->discont = FALSE; + gst_buffer_list_add (list, out_buf); } else { gst_adapter_unmap (self->adapter); From 2ab4b8ef1e73fd76cbc19ba4de3b59875b4239b2 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 18 Oct 2019 10:47:57 +0200 Subject: [PATCH 082/339] pcapparse: use inet_ntop instead of inet_ntoa inet_ntoa is not behaving properly on Mac OS, and is also deprecated. --- subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c index 29abb145760..9c6627046db 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c @@ -156,9 +156,11 @@ GST_ELEMENT_REGISTER_DEFINE (pcapparse, "pcapparse", GST_RANK_NONE, static gchar * get_ip_address_as_string (guint32 ip_addr) { + char ip_str[INET_ADDRSTRLEN]; struct in_addr addr; addr.s_addr = ip_addr; - return g_strdup (inet_ntoa (addr)); + inet_ntop (AF_INET, &addr, ip_str, INET_ADDRSTRLEN); + return g_strdup (ip_str); } static void From e596694f3aaed73c1e84b0a93dc6755eeee16a60 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 30 Jan 2020 12:50:46 +0100 Subject: [PATCH 083/339] pcapparse: include wspipdef.h for INET_ADDRSTRLEN --- subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c | 1 + 1 file changed, 1 insertion(+) diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c index 9c6627046db..cc4d520c5e7 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c @@ -58,6 +58,7 @@ #include #else #include +#include #endif From e2447806ee72993b61b2c7173578d9207b35e433 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 13 Apr 2022 23:30:51 +1000 Subject: [PATCH 084/339] pcapparse: include ws2tcpip.h for inet_ntop --- subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c | 1 + 1 file changed, 1 insertion(+) diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c index cc4d520c5e7..6b409ae0ce1 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c @@ -59,6 +59,7 @@ #else #include #include +#include #endif From 7f1e9630a60849be4f6797118c6da8430460c6b1 Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Fri, 15 Dec 2017 15:16:28 +0100 Subject: [PATCH 085/339] netsim: add 'replace-dropped-with-empty' property [pexhack] To be able to simulate packet loss between AVMCU & SfB --- .../gst-plugins-bad/gst/netsim/gstnetsim.c | 37 ++++++++++++++++++- .../gst-plugins-bad/gst/netsim/gstnetsim.h | 1 + .../gst-plugins-bad/gst/netsim/meson.build | 2 +- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c index 55853b8fd01..ddd4feded12 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c @@ -31,6 +31,7 @@ #include #include #include +#include GST_DEBUG_CATEGORY (netsim_debug); #define GST_CAT_DEFAULT (netsim_debug) @@ -66,6 +67,7 @@ enum PROP_MAX_KBPS, PROP_MAX_BUCKET_SIZE, PROP_ALLOW_REORDERING, + PROP_REPLACE_DROPPED_WITH_EMPTY }; /* these numbers are nothing but wild guesses and don't reflect any reality */ @@ -79,6 +81,7 @@ enum #define DEFAULT_MAX_KBPS -1 #define DEFAULT_MAX_BUCKET_SIZE -1 #define DEFAULT_ALLOW_REORDERING TRUE +#define DEFAULT_REPLACE_DROPPED_WITH_EMPTY FALSE static GstStaticPadTemplate gst_net_sim_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -475,18 +478,23 @@ gst_net_sim_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) { GstNetSim *netsim = GST_NET_SIM (parent); GstFlowReturn ret = GST_FLOW_OK; + gboolean dropped = FALSE; - if (!gst_net_sim_token_bucket (netsim, buf)) + if (!gst_net_sim_token_bucket (netsim, buf)) { + dropped = TRUE; goto done; + } if (netsim->drop_packets > 0) { netsim->drop_packets--; GST_DEBUG_OBJECT (netsim, "Dropping packet (%d left)", netsim->drop_packets); + dropped = TRUE; } else if (netsim->drop_probability > 0 && g_rand_double (netsim->rand_seed) < (gdouble) netsim->drop_probability) { GST_DEBUG_OBJECT (netsim, "Dropping packet"); + dropped = TRUE; } else if (netsim->duplicate_probability > 0 && g_rand_double (netsim->rand_seed) < (gdouble) netsim->duplicate_probability) { @@ -498,6 +506,18 @@ gst_net_sim_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) } done: + if (dropped && netsim->replace_droppped_with_empty) { + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + if (gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp)) { + guint header_len = gst_rtp_buffer_get_header_len (&rtp); + gst_rtp_buffer_unmap (&rtp); + + buf = gst_buffer_make_writable (buf); + gst_buffer_resize (buf, 0, header_len); + ret = gst_net_sim_delay_buffer (netsim, buf); + } + } gst_buffer_unref (buf); return ret; } @@ -542,6 +562,9 @@ gst_net_sim_set_property (GObject * object, case PROP_ALLOW_REORDERING: netsim->allow_reordering = g_value_get_boolean (value); break; + case PROP_REPLACE_DROPPED_WITH_EMPTY: + netsim->replace_droppped_with_empty = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -585,6 +608,9 @@ gst_net_sim_get_property (GObject * object, case PROP_ALLOW_REORDERING: g_value_set_boolean (value, netsim->allow_reordering); break; + case PROP_REPLACE_DROPPED_WITH_EMPTY: + g_value_set_boolean (value, netsim->replace_droppped_with_empty); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -713,6 +739,15 @@ gst_net_sim_class_init (GstNetSimClass * klass) "Drop the next n packets", 0, G_MAXUINT, DEFAULT_DROP_PACKETS, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_REPLACE_DROPPED_WITH_EMPTY, + g_param_spec_boolean ( + "replace-dropped-with-empty-packets", + "replace-dropped-with-empty-packets", + "Insert packets with no payload instead of dropping", + DEFAULT_REPLACE_DROPPED_WITH_EMPTY, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + /** * GstNetSim:max-kbps: * diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h index cbd627012d2..ace3e4c71df 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h @@ -88,6 +88,7 @@ struct _GstNetSim gint max_kbps; gint max_bucket_size; gboolean allow_reordering; + gboolean replace_droppped_with_empty; }; struct _GstNetSimClass diff --git a/subprojects/gst-plugins-bad/gst/netsim/meson.build b/subprojects/gst-plugins-bad/gst/netsim/meson.build index 27a5ca4f9f8..78ac68f3a24 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/meson.build +++ b/subprojects/gst-plugins-bad/gst/netsim/meson.build @@ -6,7 +6,7 @@ gstnetsim = library('gstnetsim', netsim_sources, c_args : gst_plugins_bad_args, include_directories : [configinc], - dependencies : [gstbase_dep, libm], + dependencies : [gstbase_dep, gstrtp_dep, libm], install : true, install_dir : plugins_install_dir, ) From c5058326884d14819a1786986a67b3a8cefdc5d4 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 12 Sep 2019 11:08:29 +0200 Subject: [PATCH 086/339] netsim: remove GMainLoop and add an internal queue To allow the token bucket algorithm to queue instead of just dropping buffers. --- .../gst-plugins-bad/gst/netsim/gstnetsim.c | 638 +++++++++++------- .../gst-plugins-bad/gst/netsim/gstnetsim.h | 22 +- 2 files changed, 398 insertions(+), 262 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c index ddd4feded12..a6ac4e384d2 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c @@ -54,6 +54,11 @@ distribution_get_type (void) return static_g_define_type_id; } +#define GST_NET_SIM_LOCK(obj) g_mutex_lock (&obj->mutex) +#define GST_NET_SIM_UNLOCK(obj) g_mutex_unlock (&obj->mutex) +#define GST_NET_SIM_SIGNAL(obj) g_cond_signal (&obj->cond) +#define GST_NET_SIM_WAIT(obj) g_cond_wait (&obj->cond, &obj->mutex) + enum { PROP_0, @@ -66,6 +71,8 @@ enum PROP_DROP_PACKETS, PROP_MAX_KBPS, PROP_MAX_BUCKET_SIZE, + PROP_QUEUE_SIZE, + PROP_MAX_QUEUE_DELAY, PROP_ALLOW_REORDERING, PROP_REPLACE_DROPPED_WITH_EMPTY }; @@ -80,6 +87,8 @@ enum #define DEFAULT_DROP_PACKETS 0 #define DEFAULT_MAX_KBPS -1 #define DEFAULT_MAX_BUCKET_SIZE -1 +#define DEFAULT_QUEUE_SIZE -1 +#define DEFAULT_MAX_QUEUE_DELAY 50 #define DEFAULT_ALLOW_REORDERING TRUE #define DEFAULT_REPLACE_DROPPED_WITH_EMPTY FALSE @@ -99,148 +108,102 @@ G_DEFINE_TYPE (GstNetSim, gst_net_sim, GST_TYPE_ELEMENT); GST_ELEMENT_REGISTER_DEFINE (netsim, "netsim", GST_RANK_MARGINAL, GST_TYPE_NET_SIM); -static gboolean -gst_net_sim_source_dispatch (GSource * source, - GSourceFunc callback, gpointer user_data) +typedef struct { - callback (user_data); - return FALSE; + GstBuffer *buf; + guint size_bits; + GstClockTime arrival_time; + GstClockTime delay; + GstClockTime token_delay; + guint seqnum; +} NetSimBuffer; + +static NetSimBuffer * +net_sim_buffer_new (GstBuffer * buf, + guint seqnum, GstClockTime arrival_time, GstClockTime delay) +{ + NetSimBuffer *nsbuf = g_new (NetSimBuffer, 1); + nsbuf->buf = gst_buffer_ref (buf); + nsbuf->size_bits = gst_buffer_get_size (buf) * 8; + nsbuf->seqnum = seqnum; + nsbuf->arrival_time = arrival_time; + nsbuf->delay = delay; + nsbuf->token_delay = 0; + return nsbuf; } -GSourceFuncs gst_net_sim_source_funcs = { - NULL, /* prepare */ - NULL, /* check */ - gst_net_sim_source_dispatch, - NULL /* finalize */ -}; - -static void -gst_net_sim_loop (GstNetSim * netsim) +static GstClockTime +net_sim_buffer_get_sync_time (NetSimBuffer * nsbuf) { - GMainLoop *loop; - - GST_TRACE_OBJECT (netsim, "TASK: begin"); - - g_mutex_lock (&netsim->loop_mutex); - loop = g_main_loop_ref (netsim->main_loop); - netsim->running = TRUE; - GST_TRACE_OBJECT (netsim, "TASK: signal start"); - g_cond_signal (&netsim->start_cond); - g_mutex_unlock (&netsim->loop_mutex); - - GST_TRACE_OBJECT (netsim, "TASK: run"); - g_main_loop_run (loop); - g_main_loop_unref (loop); - - g_mutex_lock (&netsim->loop_mutex); - GST_TRACE_OBJECT (netsim, "TASK: pause"); - gst_pad_pause_task (netsim->srcpad); - netsim->running = FALSE; - GST_TRACE_OBJECT (netsim, "TASK: signal end"); - g_cond_signal (&netsim->start_cond); - g_mutex_unlock (&netsim->loop_mutex); - GST_TRACE_OBJECT (netsim, "TASK: end"); + return nsbuf->arrival_time + nsbuf->delay + nsbuf->token_delay; } -static gboolean -_main_loop_quit_and_remove_source (gpointer user_data) +static void +net_sim_buffer_free (NetSimBuffer * nsbuf) { - GMainLoop *main_loop = user_data; - GST_DEBUG ("MAINLOOP: Quit %p", main_loop); - g_main_loop_quit (main_loop); - g_assert (!g_main_loop_is_running (main_loop)); - return FALSE; /* Remove source */ + if (G_UNLIKELY (nsbuf == NULL)) + return; + gst_buffer_unref (nsbuf->buf); + g_free (nsbuf); } -static gboolean -gst_net_sim_src_activatemode (GstPad * pad, GstObject * parent, - GstPadMode mode, gboolean active) +static GstFlowReturn +net_sim_buffer_push (NetSimBuffer * nsbuf, GstPad * srcpad) { - GstNetSim *netsim = GST_NET_SIM (parent); - gboolean result = FALSE; - - g_mutex_lock (&netsim->loop_mutex); - if (active) { - if (netsim->main_loop == NULL) { - GMainContext *main_context = g_main_context_new (); - netsim->main_loop = g_main_loop_new (main_context, FALSE); - g_main_context_unref (main_context); - - GST_TRACE_OBJECT (netsim, "ACT: Starting task on srcpad"); - result = gst_pad_start_task (netsim->srcpad, - (GstTaskFunction) gst_net_sim_loop, netsim, NULL); - - GST_TRACE_OBJECT (netsim, "ACT: Wait for task to start"); - g_assert (!netsim->running); - while (!netsim->running) - g_cond_wait (&netsim->start_cond, &netsim->loop_mutex); - GST_TRACE_OBJECT (netsim, "ACT: Task on srcpad started"); - } - } else { - if (netsim->main_loop != NULL) { - GSource *source; - guint id; - - /* Adds an Idle Source which quits the main loop from within. - * This removes the possibility for run/quit race conditions. */ - GST_TRACE_OBJECT (netsim, "DEACT: Stopping main loop on deactivate"); - source = g_idle_source_new (); - g_source_set_callback (source, _main_loop_quit_and_remove_source, - g_main_loop_ref (netsim->main_loop), - (GDestroyNotify) g_main_loop_unref); - id = g_source_attach (source, - g_main_loop_get_context (netsim->main_loop)); - g_source_unref (source); - g_assert_cmpuint (id, >, 0); - g_main_loop_unref (netsim->main_loop); - netsim->main_loop = NULL; - - GST_TRACE_OBJECT (netsim, "DEACT: Wait for mainloop and task to pause"); - g_assert (netsim->running); - while (netsim->running) - g_cond_wait (&netsim->start_cond, &netsim->loop_mutex); - - GST_TRACE_OBJECT (netsim, "DEACT: Stopping task on srcpad"); - result = gst_pad_stop_task (netsim->srcpad); - GST_TRACE_OBJECT (netsim, "DEACT: Mainloop and GstTask stopped"); - } - } - g_mutex_unlock (&netsim->loop_mutex); - - return result; + GstFlowReturn ret; + ret = gst_pad_push (srcpad, nsbuf->buf); + g_free (nsbuf); + return ret; } -typedef struct +static gint +nsbuf_compare_seqnum (gconstpointer a, gconstpointer b, + G_GNUC_UNUSED gpointer user_data) { - GstPad *pad; - GstBuffer *buf; -} PushBufferCtx; + const NetSimBuffer *buf_a = (NetSimBuffer *) a; + const NetSimBuffer *buf_b = (NetSimBuffer *) b; + return buf_a->seqnum - buf_b->seqnum; +} -static inline PushBufferCtx * -push_buffer_ctx_new (GstPad * pad, GstBuffer * buf) +static gint +nsbuf_compare_time (gconstpointer a, gconstpointer b, + G_GNUC_UNUSED gpointer user_data) { - PushBufferCtx *ctx = g_new (PushBufferCtx, 1); - ctx->pad = gst_object_ref (pad); - ctx->buf = gst_buffer_ref (buf); - return ctx; + const NetSimBuffer *buf_a = (NetSimBuffer *) a; + const NetSimBuffer *buf_b = (NetSimBuffer *) b; + GstClockTime ts_a = buf_a->arrival_time + buf_a->delay; + GstClockTime ts_b = buf_b->arrival_time + buf_b->delay; + return ts_a - ts_b; } -static inline void -push_buffer_ctx_free (PushBufferCtx * ctx) +static gboolean +gst_net_sim_set_clock (GstElement * element, GstClock * clock) { - if (G_LIKELY (ctx != NULL)) { - gst_buffer_unref (ctx->buf); - gst_object_unref (ctx->pad); - g_free (ctx); - } + GstNetSim *netsim = GST_NET_SIM_CAST (element); + GST_DEBUG_OBJECT (netsim, "Setting clock %" GST_PTR_FORMAT, clock); + if (clock == NULL) + return TRUE; + + GST_NET_SIM_LOCK (netsim); + if (netsim->clock) + gst_object_unref (netsim->clock); + netsim->clock = gst_object_ref (clock); + GST_NET_SIM_SIGNAL (netsim); + GST_NET_SIM_UNLOCK (netsim); + + return TRUE; } static gboolean -push_buffer_ctx_push (PushBufferCtx * ctx) +gst_net_sim_wait_for_clock (GstNetSim * netsim) { - GST_DEBUG_OBJECT (ctx->pad, "Pushing buffer now"); - gst_pad_push (ctx->pad, gst_buffer_ref (ctx->buf)); - return FALSE; + while (netsim->clock == NULL) { + if (!netsim->running) + return FALSE; + GST_INFO_OBJECT (netsim, "Waiting for a clock"); + GST_NET_SIM_WAIT (netsim); + } + return TRUE; } static gint @@ -331,160 +294,196 @@ get_random_value_gamma (GRand * rand_seed, gint32 low, gint32 high, return round (x + low); } -static GstFlowReturn -gst_net_sim_delay_buffer (GstNetSim * netsim, GstBuffer * buf) -{ - GstFlowReturn ret = GST_FLOW_OK; - - g_mutex_lock (&netsim->loop_mutex); - if (netsim->main_loop != NULL && netsim->delay_probability > 0 && - g_rand_double (netsim->rand_seed) < netsim->delay_probability) { - gint delay; - PushBufferCtx *ctx; - GSource *source; - gint64 ready_time, now_time; - - switch (netsim->delay_distribution) { - case DISTRIBUTION_UNIFORM: - delay = get_random_value_uniform (netsim->rand_seed, netsim->min_delay, - netsim->max_delay); - break; - case DISTRIBUTION_NORMAL: - delay = get_random_value_normal (netsim->rand_seed, netsim->min_delay, - netsim->max_delay, &netsim->delay_state); - break; - case DISTRIBUTION_GAMMA: - delay = get_random_value_gamma (netsim->rand_seed, netsim->min_delay, - netsim->max_delay, &netsim->delay_state); - break; - default: - g_assert_not_reached (); - break; - } - - if (delay < 0) - delay = 0; - - ctx = push_buffer_ctx_new (netsim->srcpad, buf); - - source = g_source_new (&gst_net_sim_source_funcs, sizeof (GSource)); - now_time = g_get_monotonic_time (); - ready_time = now_time + delay * 1000; - if (!netsim->allow_reordering && ready_time < netsim->last_ready_time) - ready_time = netsim->last_ready_time + 1; - - netsim->last_ready_time = ready_time; - GST_DEBUG_OBJECT (netsim, "Delaying packet by %" G_GINT64_FORMAT "ms", - (ready_time - now_time) / 1000); - - g_source_set_ready_time (source, ready_time); - g_source_set_callback (source, (GSourceFunc) push_buffer_ctx_push, - ctx, (GDestroyNotify) push_buffer_ctx_free); - g_source_attach (source, g_main_loop_get_context (netsim->main_loop)); - g_source_unref (source); - } else { - ret = gst_pad_push (netsim->srcpad, gst_buffer_ref (buf)); - } - g_mutex_unlock (&netsim->loop_mutex); - - return ret; -} - static gint -gst_net_sim_get_tokens (GstNetSim * netsim) +gst_net_sim_get_tokens (GstNetSim * netsim, GstClockTime now) { gint tokens = 0; GstClockTimeDiff elapsed_time = 0; - GstClockTime current_time = 0; GstClockTimeDiff token_time; - GstClock *clock; + guint max_bps; /* check for umlimited kbps and fill up the bucket if that is the case, * if not, calculate the number of tokens to add based on the elapsed time */ if (netsim->max_kbps == -1) return netsim->max_bucket_size * 1000 - netsim->bucket_size; - /* get the current time */ - clock = gst_element_get_clock (GST_ELEMENT_CAST (netsim)); - if (clock == NULL) { - GST_WARNING_OBJECT (netsim, "No clock, can't get the time"); - } else { - current_time = gst_clock_get_time (clock); - } - /* get the elapsed time */ if (GST_CLOCK_TIME_IS_VALID (netsim->prev_time)) { - if (current_time < netsim->prev_time) { + if (now < netsim->prev_time) { GST_WARNING_OBJECT (netsim, "Clock is going backwards!!"); } else { - elapsed_time = GST_CLOCK_DIFF (netsim->prev_time, current_time); + elapsed_time = GST_CLOCK_DIFF (netsim->prev_time, now); } } else { - netsim->prev_time = current_time; + netsim->prev_time = now; } /* calculate number of tokens and how much time is "spent" by these tokens */ - tokens = - gst_util_uint64_scale_int (elapsed_time, netsim->max_kbps * 1000, - GST_SECOND); - token_time = - gst_util_uint64_scale_int (GST_SECOND, tokens, netsim->max_kbps * 1000); + max_bps = netsim->max_kbps * 1000; + tokens = gst_util_uint64_scale_int (elapsed_time, max_bps, GST_SECOND); + token_time = gst_util_uint64_scale_int (GST_SECOND, tokens, max_bps); + + GST_DEBUG_OBJECT (netsim, + "Elapsed time: %" GST_TIME_FORMAT " produces %u tokens (" "token-time: %" + GST_TIME_FORMAT ")", GST_TIME_ARGS (elapsed_time), tokens, + GST_TIME_ARGS (token_time)); /* increment the time with how much we spent in terms of whole tokens */ netsim->prev_time += token_time; - gst_object_unref (clock); return tokens; } -static gboolean -gst_net_sim_token_bucket (GstNetSim * netsim, GstBuffer * buf) +static guint +gst_net_sim_get_missing_tokens (GstNetSim * netsim, + NetSimBuffer * nsbuf, GstClockTime now) { - gsize buffer_size; gint tokens; /* with an unlimited bucket-size, we have nothing to do */ if (netsim->max_bucket_size == -1) - return TRUE; + return 0; - /* get buffer size in bits */ - buffer_size = gst_buffer_get_size (buf) * 8; - tokens = gst_net_sim_get_tokens (netsim); + tokens = gst_net_sim_get_tokens (netsim, now); - netsim->bucket_size = MIN (G_MAXINT, netsim->bucket_size + tokens); - GST_LOG_OBJECT (netsim, - "Adding %d tokens to bucket (contains %" G_GSIZE_FORMAT " tokens)", + netsim->bucket_size = + MIN (netsim->max_bucket_size * 1000, netsim->bucket_size + tokens); + GST_LOG_OBJECT (netsim, "Added %d tokens to bucket (contains %u tokens)", tokens, netsim->bucket_size); - if (netsim->max_bucket_size != -1 && netsim->bucket_size > - netsim->max_bucket_size * 1000) - netsim->bucket_size = netsim->max_bucket_size * 1000; + if (nsbuf->size_bits > netsim->bucket_size) { + GST_DEBUG_OBJECT (netsim, "Buffer size (%u) exeedes bucket size (%u)", + nsbuf->size_bits, netsim->bucket_size); + return nsbuf->size_bits - netsim->bucket_size; + } + + netsim->bucket_size -= nsbuf->size_bits; + GST_DEBUG_OBJECT (netsim, "Buffer taking %u tokens (%u left)", + nsbuf->size_bits, netsim->bucket_size); + return 0; +} + +static void +gst_net_sim_drop_nsbuf (GstNetSim * netsim) +{ + NetSimBuffer *nsbuf; + nsbuf = g_queue_pop_head (netsim->bqueue); + GST_LOG_OBJECT (netsim, "Dropping buf #%u (tokens)", nsbuf->seqnum); + netsim->bits_in_queue -= nsbuf->size_bits; + net_sim_buffer_free (nsbuf); +} + +/* must be called with GST_NET_SIM_LOCK */ +static GstFlowReturn +gst_net_sim_push_unlocked (GstNetSim * netsim, GstClockTimeDiff jitter) +{ + GstFlowReturn ret = GST_FLOW_OK; + GstClockTime now = gst_clock_get_time (netsim->clock); + NetSimBuffer *nsbuf = g_queue_peek_head (netsim->bqueue); + GstClockTime sync_time = net_sim_buffer_get_sync_time (nsbuf); + guint missing_tokens; + + GST_DEBUG_OBJECT (netsim, + "now: %" GST_TIME_FORMAT " jitter %" GST_STIME_FORMAT + " netsim->max_delay %" GST_STIME_FORMAT, GST_TIME_ARGS (now), + GST_STIME_ARGS (jitter), + GST_STIME_ARGS (netsim->max_delay * GST_MSECOND)); + + missing_tokens = gst_net_sim_get_missing_tokens (netsim, nsbuf, now); + if (missing_tokens > 0) { + GstClockTime token_delay = gst_util_uint64_scale_int (GST_SECOND, + missing_tokens, netsim->max_kbps * 1000); + GstClockTime new_synctime = now + token_delay; + GstClockTime delta = new_synctime - net_sim_buffer_get_sync_time (nsbuf); + nsbuf->token_delay += delta; - if (buffer_size > netsim->bucket_size) { GST_DEBUG_OBJECT (netsim, - "Buffer size (%" G_GSIZE_FORMAT ") exeedes bucket size (%" - G_GSIZE_FORMAT ")", buffer_size, netsim->bucket_size); - return FALSE; + "Missing %u tokens, delaying buffer #%u additional %" GST_TIME_FORMAT + " (total: %" GST_TIME_FORMAT ") for new sync_time: %" GST_TIME_FORMAT, + missing_tokens, nsbuf->seqnum, GST_TIME_ARGS (delta), + GST_TIME_ARGS (nsbuf->token_delay), GST_TIME_ARGS (sync_time)); + + if (nsbuf->token_delay > netsim->max_queue_delay * GST_MSECOND) { + gst_net_sim_drop_nsbuf (netsim); + } + } else { + GST_DEBUG_OBJECT (netsim, "Pushing buffer #%u now", nsbuf->seqnum); + nsbuf = g_queue_pop_head (netsim->bqueue); + netsim->bits_in_queue -= nsbuf->size_bits; + + GST_NET_SIM_UNLOCK (netsim); + ret = net_sim_buffer_push (nsbuf, netsim->srcpad); + GST_NET_SIM_LOCK (netsim); } - netsim->bucket_size -= buffer_size; - GST_LOG_OBJECT (netsim, - "Buffer taking %" G_GSIZE_FORMAT " tokens (%" G_GSIZE_FORMAT " left)", - buffer_size, netsim->bucket_size); - return TRUE; + return ret; +} + +static gint +gst_new_sim_get_delay_ms (GstNetSim * netsim) +{ + gint delay_ms = 0; + + if (netsim->delay_probability == 0 || + g_rand_double (netsim->rand_seed) > netsim->delay_probability) + return delay_ms; + + switch (netsim->delay_distribution) { + case DISTRIBUTION_UNIFORM: + delay_ms = get_random_value_uniform (netsim->rand_seed, netsim->min_delay, + netsim->max_delay); + break; + case DISTRIBUTION_NORMAL: + delay_ms = get_random_value_normal (netsim->rand_seed, netsim->min_delay, + netsim->max_delay, &netsim->delay_state); + break; + case DISTRIBUTION_GAMMA: + delay_ms = get_random_value_gamma (netsim->rand_seed, netsim->min_delay, + netsim->max_delay, &netsim->delay_state); + break; + default: + g_assert_not_reached (); + break; + } + + return delay_ms; +} + +static void +gst_net_sim_queue_buffer (GstNetSim * netsim, GstBuffer * buf) +{ + GstClockTime now = gst_clock_get_time (netsim->clock); + gint delay_ms = gst_new_sim_get_delay_ms (netsim); + NetSimBuffer *nsbuf = net_sim_buffer_new (buf, netsim->seqnum++, + now, delay_ms * GST_MSECOND); + + if (delay_ms > 0) { + GST_DEBUG_OBJECT (netsim, "Delaying buffer with %dms", delay_ms); + } + + GST_DEBUG_OBJECT (netsim, "queue_size: %u, bits_in_queue: %u, bufsize: %u", + netsim->queue_size * 1000, netsim->bits_in_queue, nsbuf->size_bits); + + if (netsim->bits_in_queue > 0 && + netsim->queue_size != -1 && + netsim->bits_in_queue + nsbuf->size_bits > netsim->queue_size * 1000) { + GST_DEBUG_OBJECT (netsim, "dropping buf #%u", nsbuf->seqnum); + net_sim_buffer_free (nsbuf); + } else { + GST_DEBUG_OBJECT (netsim, "queueing buf #%u", nsbuf->seqnum); + GST_NET_SIM_LOCK (netsim); + g_queue_insert_sorted (netsim->bqueue, nsbuf, netsim->compare_func, NULL); + netsim->bits_in_queue += nsbuf->size_bits; + GST_NET_SIM_SIGNAL (netsim); + GST_NET_SIM_UNLOCK (netsim); + } } static GstFlowReturn gst_net_sim_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) { - GstNetSim *netsim = GST_NET_SIM (parent); - GstFlowReturn ret = GST_FLOW_OK; + GstNetSim *netsim = GST_NET_SIM_CAST (parent); gboolean dropped = FALSE; - if (!gst_net_sim_token_bucket (netsim, buf)) { - dropped = TRUE; - goto done; - } - if (netsim->drop_packets > 0) { netsim->drop_packets--; GST_DEBUG_OBJECT (netsim, "Dropping packet (%d left)", @@ -499,13 +498,12 @@ gst_net_sim_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) g_rand_double (netsim->rand_seed) < (gdouble) netsim->duplicate_probability) { GST_DEBUG_OBJECT (netsim, "Duplicating packet"); - gst_net_sim_delay_buffer (netsim, buf); - ret = gst_net_sim_delay_buffer (netsim, buf); + gst_net_sim_queue_buffer (netsim, buf); + gst_net_sim_queue_buffer (netsim, buf); } else { - ret = gst_net_sim_delay_buffer (netsim, buf); + gst_net_sim_queue_buffer (netsim, buf); } -done: if (dropped && netsim->replace_droppped_with_empty) { GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; @@ -515,19 +513,111 @@ gst_net_sim_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) buf = gst_buffer_make_writable (buf); gst_buffer_resize (buf, 0, header_len); - ret = gst_net_sim_delay_buffer (netsim, buf); + gst_net_sim_queue_buffer (netsim, buf); } } + gst_buffer_unref (buf); - return ret; + + return GST_FLOW_OK; } +static void +gst_net_sim_loop (gpointer data) +{ + GstNetSim *netsim = GST_NET_SIM_CAST (data); + GstFlowReturn ret; + NetSimBuffer *nsbuf; + GstClockTime sync_time; + GstClockTimeDiff jitter; + + GST_NET_SIM_LOCK (netsim); + if (!gst_net_sim_wait_for_clock (netsim)) + goto pause_task; + + if (!netsim->running) + goto pause_task; + + while (netsim->running && g_queue_is_empty (netsim->bqueue)) + GST_NET_SIM_WAIT (netsim); + + if (!netsim->running) + goto pause_task; + + nsbuf = g_queue_peek_head (netsim->bqueue); + sync_time = net_sim_buffer_get_sync_time (nsbuf); + netsim->clock_id = gst_clock_new_single_shot_id (netsim->clock, sync_time); + + GST_DEBUG_OBJECT (netsim, "Popped buf #%u with sync_time: %" GST_TIME_FORMAT, + nsbuf->seqnum, GST_TIME_ARGS (sync_time)); + + GST_NET_SIM_UNLOCK (netsim); + gst_clock_id_wait (netsim->clock_id, &jitter); + GST_NET_SIM_LOCK (netsim); + + gst_clock_id_unref (netsim->clock_id); + netsim->clock_id = NULL; + + if (!netsim->running) { + goto pause_task; + } + + ret = gst_net_sim_push_unlocked (netsim, jitter); + if (ret != GST_FLOW_OK) { + GST_ERROR_OBJECT (netsim, "pausing task because flow: %d", ret); + goto pause_task; + } + + GST_NET_SIM_UNLOCK (netsim); + return; + +pause_task: + GST_INFO_OBJECT (netsim, "pausing task"); + gst_pad_pause_task (netsim->srcpad); + GST_NET_SIM_UNLOCK (netsim); + return; +} + +static gboolean +gst_net_sim_src_activatemode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + GstNetSim *netsim = GST_NET_SIM_CAST (parent); + gboolean result; + + if (active) { + GST_DEBUG_OBJECT (pad, "starting task on pad %s:%s", + GST_DEBUG_PAD_NAME (pad)); + + GST_NET_SIM_LOCK (netsim); + netsim->running = TRUE; + if (!gst_pad_start_task (netsim->srcpad, gst_net_sim_loop, netsim, NULL)) + g_assert_not_reached (); + + GST_NET_SIM_UNLOCK (netsim); + result = TRUE; + } else { + GST_DEBUG_OBJECT (pad, "stopping task on pad %s:%s", + GST_DEBUG_PAD_NAME (pad)); + + GST_NET_SIM_LOCK (netsim); + netsim->running = FALSE; + if (netsim->clock_id) + gst_clock_id_unschedule (netsim->clock_id); + GST_NET_SIM_SIGNAL (netsim); + GST_NET_SIM_UNLOCK (netsim); + + result = gst_pad_stop_task (pad); + } + + return result; +} static void gst_net_sim_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - GstNetSim *netsim = GST_NET_SIM (object); + GstNetSim *netsim = GST_NET_SIM_CAST (object); switch (prop_id) { case PROP_MIN_DELAY: @@ -559,8 +649,18 @@ gst_net_sim_set_property (GObject * object, if (netsim->max_bucket_size != -1) netsim->bucket_size = netsim->max_bucket_size * 1000; break; + case PROP_QUEUE_SIZE: + netsim->queue_size = g_value_get_int (value); + break; + case PROP_MAX_QUEUE_DELAY: + netsim->max_queue_delay = g_value_get_int (value); + break; case PROP_ALLOW_REORDERING: netsim->allow_reordering = g_value_get_boolean (value); + if (netsim->allow_reordering) + netsim->compare_func = nsbuf_compare_time; + else + netsim->compare_func = nsbuf_compare_seqnum; break; case PROP_REPLACE_DROPPED_WITH_EMPTY: netsim->replace_droppped_with_empty = g_value_get_boolean (value); @@ -575,7 +675,7 @@ static void gst_net_sim_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - GstNetSim *netsim = GST_NET_SIM (object); + GstNetSim *netsim = GST_NET_SIM_CAST (object); switch (prop_id) { case PROP_MIN_DELAY: @@ -605,6 +705,12 @@ gst_net_sim_get_property (GObject * object, case PROP_MAX_BUCKET_SIZE: g_value_set_int (value, netsim->max_bucket_size); break; + case PROP_QUEUE_SIZE: + g_value_set_int (value, netsim->queue_size); + break; + case PROP_MAX_QUEUE_DELAY: + g_value_set_int (value, netsim->max_queue_delay); + break; case PROP_ALLOW_REORDERING: g_value_set_boolean (value, netsim->allow_reordering); break; @@ -629,10 +735,10 @@ gst_net_sim_init (GstNetSim * netsim) gst_element_add_pad (GST_ELEMENT (netsim), netsim->srcpad); gst_element_add_pad (GST_ELEMENT (netsim), netsim->sinkpad); - g_mutex_init (&netsim->loop_mutex); - g_cond_init (&netsim->start_cond); + netsim->bqueue = g_queue_new (); + g_mutex_init (&netsim->mutex); + g_cond_init (&netsim->cond); netsim->rand_seed = g_rand_new (); - netsim->main_loop = NULL; netsim->prev_time = GST_CLOCK_TIME_NONE; GST_OBJECT_FLAG_SET (netsim->sinkpad, @@ -647,23 +753,17 @@ gst_net_sim_init (GstNetSim * netsim) static void gst_net_sim_finalize (GObject * object) { - GstNetSim *netsim = GST_NET_SIM (object); + GstNetSim *netsim = GST_NET_SIM_CAST (object); g_rand_free (netsim->rand_seed); - g_mutex_clear (&netsim->loop_mutex); - g_cond_clear (&netsim->start_cond); + g_mutex_clear (&netsim->mutex); + g_cond_clear (&netsim->cond); - G_OBJECT_CLASS (gst_net_sim_parent_class)->finalize (object); -} - -static void -gst_net_sim_dispose (GObject * object) -{ - GstNetSim *netsim = GST_NET_SIM (object); + gst_object_replace ((GstObject **) & netsim->clock, NULL); - g_assert (netsim->main_loop == NULL); + g_queue_free_full (netsim->bqueue, (GDestroyNotify) net_sim_buffer_free); - G_OBJECT_CLASS (gst_net_sim_parent_class)->dispose (object); + G_OBJECT_CLASS (gst_net_sim_parent_class)->finalize (object); } static void @@ -685,11 +785,11 @@ gst_net_sim_class_init (GstNetSimClass * klass) "Philippe Kalaf , " "Havard Graff "); - gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_net_sim_dispose); gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_net_sim_finalize); gobject_class->set_property = gst_net_sim_set_property; gobject_class->get_property = gst_net_sim_get_property; + gstelement_class->set_clock = GST_DEBUG_FUNCPTR (gst_net_sim_set_clock); g_object_class_install_property (gobject_class, PROP_MIN_DELAY, g_param_spec_int ("min-delay", "Minimum delay (ms)", @@ -740,9 +840,9 @@ gst_net_sim_class_init (GstNetSimClass * klass) 0, G_MAXUINT, DEFAULT_DROP_PACKETS, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_REPLACE_DROPPED_WITH_EMPTY, - g_param_spec_boolean ( - "replace-dropped-with-empty-packets", + g_object_class_install_property (gobject_class, + PROP_REPLACE_DROPPED_WITH_EMPTY, + g_param_spec_boolean ("replace-dropped-with-empty-packets", "replace-dropped-with-empty-packets", "Insert packets with no payload instead of dropping", DEFAULT_REPLACE_DROPPED_WITH_EMPTY, @@ -776,6 +876,34 @@ gst_net_sim_class_init (GstNetSimClass * klass) "(-1 = unlimited)", -1, G_MAXINT, DEFAULT_MAX_BUCKET_SIZE, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + /** + * GstNetSim:queue-size: + * + * In case of insufficient tokens in the bucket, this property says how + * much bits we can keep in the queue + * + * Since: 1.18 + */ + g_object_class_install_property (gobject_class, PROP_QUEUE_SIZE, + g_param_spec_int ("queue-size", "Queue Size in bits", + "The max number of bits in the internal queue " + "(-1 = unlimited)", + -1, G_MAXINT, DEFAULT_QUEUE_SIZE, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + /** + * GstNetSim:max-queue-delay: + * + * How long we will delay packets for in the queue + * + * Since: 1.18 + */ + g_object_class_install_property (gobject_class, PROP_MAX_QUEUE_DELAY, + g_param_spec_int ("max-queue-delay", "Maximum queue delay (ms)", + "The maximum delay a buffer can be given before being dropped", + G_MININT, G_MAXINT, DEFAULT_MAX_QUEUE_DELAY, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + /** * GstNetSim:allow-reordering: * diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h index ace3e4c71df..c5dcaa4cc54 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h @@ -42,6 +42,7 @@ G_BEGIN_DECLS (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_NET_SIM)) #define GST_IS_NET_SIM_CLASS(obj) \ (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_NET_SIM)) +#define GST_NET_SIM_CAST(obj) ((GstNetSim *)obj) typedef struct _GstNetSim GstNetSim; typedef struct _GstNetSimClass GstNetSimClass; @@ -67,15 +68,20 @@ struct _GstNetSim GstPad *sinkpad; GstPad *srcpad; - GMutex loop_mutex; - GCond start_cond; - GMainLoop *main_loop; - gboolean running; GRand *rand_seed; - gsize bucket_size; - GstClockTime prev_time; + guint bucket_size; NormalDistributionState delay_state; - gint64 last_ready_time; + GstClockTime prev_time; + + GstClock *clock; + GstClockID clock_id; + GMutex mutex; + GCond cond; + gboolean running; + GQueue *bqueue; + guint seqnum; + GCompareDataFunc compare_func; + guint bits_in_queue; /* properties */ gint min_delay; @@ -87,6 +93,8 @@ struct _GstNetSim guint drop_packets; gint max_kbps; gint max_bucket_size; + gint queue_size; + gint max_queue_delay; gboolean allow_reordering; gboolean replace_droppped_with_empty; }; From b36277c338729dee611a94c61a1134de1bd49013 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 1 Jun 2018 16:46:19 +0200 Subject: [PATCH 087/339] Initial commit of gsthomograff --- .../ext/opencv/gsthomography.cpp | 301 ++++++++++++++++++ .../ext/opencv/gsthomography.h | 81 +++++ .../gst-plugins-bad/ext/opencv/gstopencv.cpp | 2 + .../gst-plugins-bad/ext/opencv/meson.build | 1 + 4 files changed, 385 insertions(+) create mode 100644 subprojects/gst-plugins-bad/ext/opencv/gsthomography.cpp create mode 100644 subprojects/gst-plugins-bad/ext/opencv/gsthomography.h diff --git a/subprojects/gst-plugins-bad/ext/opencv/gsthomography.cpp b/subprojects/gst-plugins-bad/ext/opencv/gsthomography.cpp new file mode 100644 index 00000000000..bdae2ad838c --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/opencv/gsthomography.cpp @@ -0,0 +1,301 @@ +/* + * GStreamer + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Alternatively, the contents of this file may be used under the + * GNU Lesser General Public License Version 2.1 (the "LGPL"), in + * which case the following provisions apply instead of the ones + * mentioned above: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include "gsthomography.h" +#include +#include + +GST_DEBUG_CATEGORY_STATIC (gst_homography_debug); +#define GST_CAT_DEFAULT gst_homography_debug + +/* Filter signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +enum +{ + PROP_0, +}; + +/* the capabilities of the inputs and outputs. + * + * describe the real formats here. + */ +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGB")) + ); + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGB")) + ); + +G_DEFINE_TYPE (GstHomography, gst_homography, GST_TYPE_OPENCV_VIDEO_FILTER); + +static void gst_homography_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_homography_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstFlowReturn gst_homography_transform (GstOpencvVideoFilter * filter, + GstBuffer * buf, IplImage * img, GstBuffer * outbuf, IplImage * outimg); + +static void +gst_homography_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstHomography *filter = GST_HOMOGRAPHY (object); + (void)filter; + (void)value; + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_homography_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstHomography *filter = GST_HOMOGRAPHY (object); + (void)filter; + (void)value; + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GstCaps * +gst_homography_transform_caps (GstBaseTransform * trans, + GstPadDirection dir, GstCaps * caps, GstCaps * filter) +{ + GstCaps * ret = gst_caps_copy (caps); + guint i; + + GST_DEBUG_OBJECT (trans, + "transforming caps %"GST_PTR_FORMAT" filter %"GST_PTR_FORMAT" in direction %s", + caps, filter, (dir == GST_PAD_SINK) ? "sink" : "src"); + + for (i = 0; i < gst_caps_get_size (ret); ++i) { + GstStructure * s = gst_caps_get_structure (ret, i); + + gst_structure_set (s, + "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL); + } + + if (!gst_caps_is_empty (ret) && filter) { + GstCaps *tmp = gst_caps_intersect_full (filter, ret, + GST_CAPS_INTERSECT_FIRST); + gst_caps_replace (&ret, tmp); + gst_caps_unref (tmp); + } + + GST_DEBUG_OBJECT (trans, "returning caps %" GST_PTR_FORMAT, ret); + + return ret; +} + + +static float +euclidean_distance (CvPoint2D32f * a, CvPoint2D32f * b) +{ + CvPoint2D32f d; + d.x = a->x - b->x; + d.y = a->y - b->y; + + return sqrtf (d.x * d.x + d.y * d.y); +} + +static GstCaps * +gst_homography_fixate_caps (GstBaseTransform * trans, + GstPadDirection dir, GstCaps * caps, GstCaps * othercaps) +{ + GstHomography *filter = GST_HOMOGRAPHY (trans); + GstCaps * ret; + GstStructure * out_s; + + g_return_val_if_fail (gst_caps_is_fixed (caps), NULL); + + GST_DEBUG_OBJECT (filter, + "fixating othercaps %" GST_PTR_FORMAT " in direction %s based on %"GST_PTR_FORMAT, + othercaps, (dir == GST_PAD_SINK) ? "sink" : "src", caps); + + ret = gst_caps_intersect (othercaps, caps); + if (gst_caps_is_empty (ret)) { + gst_caps_unref (ret); + ret = gst_caps_ref (othercaps); + } + + ret = gst_caps_make_writable (ret); + out_s = gst_caps_get_structure (ret, 0); + + float top = euclidean_distance (&filter->src_points[0], &filter->src_points[1]); + float right = euclidean_distance (&filter->src_points[1], &filter->src_points[2]); + float bottom = euclidean_distance (&filter->src_points[2], &filter->src_points[3]); + float left = euclidean_distance (&filter->src_points[3], &filter->src_points[0]); + + //float aspect_ratio = (top + bottom) / 2.0f + (right + left) / 2.0f; + + gint max_width = (gint)round (MAX (top, bottom)); + gint max_height = (gint)round (MAX (right, left)); + + gst_structure_set (out_s, + "width", G_TYPE_INT, max_width, + "height", G_TYPE_INT, max_height, + NULL); + + return ret; +} + +static GstFlowReturn +gst_homography_transform (GstOpencvVideoFilter * base, GstBuffer * buf, + IplImage * img, GstBuffer * outbuf, IplImage * outimg) +{ + GstHomography *filter = GST_HOMOGRAPHY (base); + + CvPoint2D32f dst_points[4]; + + dst_points[0].x = 0; + dst_points[0].y = 0; + dst_points[1].x = outimg->width - 1; + dst_points[1].y = 0; + dst_points[2].x = outimg->width - 1; + dst_points[2].y = outimg->height - 1; + dst_points[3].x = 0; + dst_points[3].y = outimg->height - 1; + + CvMat src_mat = cvMat (1, 4, CV_32FC2, &filter->src_points); + CvMat dst_mat = cvMat (1, 4, CV_32FC2, &dst_points); + + double H[9]; + CvMat res_mat = cvMat (3, 3, CV_64F, H); + + cvFindHomography(&src_mat, &dst_mat, &res_mat, CV_RANSAC); + + cvWarpPerspective (img, outimg, &res_mat); + + return GST_FLOW_OK; +} + + +/* initialize the homography's class */ +static void +gst_homography_class_init (GstHomographyClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseTransformClass *trans_class = (GstBaseTransformClass *) klass; + GstOpencvVideoFilterClass *gstopencvbasefilter_class = (GstOpencvVideoFilterClass *) klass; + + gobject_class->set_property = gst_homography_set_property; + gobject_class->get_property = gst_homography_get_property; + + trans_class->transform_caps = GST_DEBUG_FUNCPTR (gst_homography_transform_caps); + trans_class->fixate_caps = GST_DEBUG_FUNCPTR (gst_homography_fixate_caps); + + gstopencvbasefilter_class->cv_trans_func = gst_homography_transform; + + gst_element_class_set_static_metadata (element_class, + "homography", + "Filter/Effect/Video", + "Performs homography.", + "Michael Sheldon "); + + gst_element_class_add_static_pad_template (element_class, &src_factory); + gst_element_class_add_static_pad_template (element_class, &sink_factory); +} + +/* initialize the new element + * instantiate pads and add them to element + * set pad calback functions + * initialize instance structure + */ +static void +gst_homography_init (GstHomography * filter) +{ + filter->src_points[0].x = 740; + filter->src_points[0].y = 187; + + filter->src_points[1].x = 1043; + filter->src_points[1].y = 165; + + filter->src_points[2].x = 1115; + filter->src_points[2].y = 604; + + filter->src_points[3].x = 771; + filter->src_points[3].y = 616; + + gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER_CAST (filter), + FALSE); +} + + +/* entry point to initialize the plug-in + * initialize the plug-in itself + * register the element factories and other features + */ +gboolean +gst_homography_plugin_init (GstPlugin * plugin) +{ + /* debug category for fltering log messages */ + GST_DEBUG_CATEGORY_INIT (gst_homography_debug, "homography", + 0, "Performs homography on videos and images"); + + return gst_element_register (plugin, "homography", GST_RANK_NONE, + GST_TYPE_HOMOGRAPHY); +} diff --git a/subprojects/gst-plugins-bad/ext/opencv/gsthomography.h b/subprojects/gst-plugins-bad/ext/opencv/gsthomography.h new file mode 100644 index 00000000000..44cd2430d54 --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/opencv/gsthomography.h @@ -0,0 +1,81 @@ +/* + * GStreamer + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Alternatively, the contents of this file may be used under the + * GNU Lesser General Public License Version 2.1 (the "LGPL"), in + * which case the following provisions apply instead of the ones + * mentioned above: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_HOMOGRAPHY_H__ +#define __GST_HOMOGRAPHY_H__ + +#include +#include + +G_BEGIN_DECLS +/* #defines don't like whitespacey bits */ +#define GST_TYPE_HOMOGRAPHY \ + (gst_homography_get_type()) +#define GST_HOMOGRAPHY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_HOMOGRAPHY,GstHomography)) +#define GST_HOMOGRAPHY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_HOMOGRAPHY,GstHomographyClass)) +#define GST_IS_HOMOGRAPHY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_HOMOGRAPHY)) +#define GST_IS_HOMOGRAPHY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_HOMOGRAPHY)) + +typedef struct _GstHomography GstHomography; +typedef struct _GstHomographyClass GstHomographyClass; + +struct _GstHomography +{ + GstOpencvVideoFilter element; + CvPoint2D32f src_points[4]; +}; + +struct _GstHomographyClass +{ + GstOpencvVideoFilterClass parent_class; +}; + +GType gst_homography_get_type (void); + +gboolean gst_homography_plugin_init (GstPlugin * plugin); + +G_END_DECLS +#endif /* __GST_HOMOGRAPHY_H__ */ diff --git a/subprojects/gst-plugins-bad/ext/opencv/gstopencv.cpp b/subprojects/gst-plugins-bad/ext/opencv/gstopencv.cpp index 5407a5a997c..70f85586bfc 100644 --- a/subprojects/gst-plugins-bad/ext/opencv/gstopencv.cpp +++ b/subprojects/gst-plugins-bad/ext/opencv/gstopencv.cpp @@ -32,6 +32,7 @@ #include "gstedgedetect.h" #include "gstfaceblur.h" #include "gstfacedetect.h" +#include "gsthomography.h" #include "gstmotioncells.h" #include "gsttemplatematch.h" #include "gsttextoverlay.h" @@ -61,6 +62,7 @@ plugin_init (GstPlugin * plugin) ret |= GST_ELEMENT_REGISTER (edgedetect, plugin); ret |= GST_ELEMENT_REGISTER (faceblur, plugin); ret |= GST_ELEMENT_REGISTER (facedetect, plugin); + ret |= GST_ELEMENT_REGISTER (homography, plugin); ret |= GST_ELEMENT_REGISTER (motioncells, plugin); ret |= GST_ELEMENT_REGISTER (templatematch, plugin); ret |= GST_ELEMENT_REGISTER (opencvtextoverlay, plugin); diff --git a/subprojects/gst-plugins-bad/ext/opencv/meson.build b/subprojects/gst-plugins-bad/ext/opencv/meson.build index 37e20156eb6..da88cff4a67 100644 --- a/subprojects/gst-plugins-bad/ext/opencv/meson.build +++ b/subprojects/gst-plugins-bad/ext/opencv/meson.build @@ -17,6 +17,7 @@ gstopencv_sources = [ 'gstfacedetect.cpp', 'gstgrabcut.cpp', 'gsthanddetect.cpp', + 'gsthomography.cpp', 'gstmotioncells.cpp', 'gstopencv.cpp', 'gstretinex.cpp', From baf30153400936eaa2876888af3848798b199f60 Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Wed, 4 Sep 2013 10:58:21 +0200 Subject: [PATCH 088/339] task: Add set_default_task_pool_type API. https://bugzilla.gnome.org/show_bug.cgi?id=773087 --- subprojects/gstreamer/gst/gsttask.c | 24 +++++++++++++++++-- subprojects/gstreamer/gst/gsttask.h | 4 ++++ .../gstreamer/tests/check/gst/struct_x86_64.h | 2 +- .../tests/check/gst/struct_x86_64w.h | 2 +- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/subprojects/gstreamer/gst/gsttask.c b/subprojects/gstreamer/gst/gsttask.c index c1356450844..4e94f1a633f 100644 --- a/subprojects/gstreamer/gst/gsttask.c +++ b/subprojects/gstreamer/gst/gsttask.c @@ -230,8 +230,17 @@ G_DEFINE_TYPE_WITH_CODE (GstTask, gst_task, GST_TYPE_OBJECT, static void ensure_klass_pool (GstTaskClass * klass) { - if (G_UNLIKELY (_global_task_pool == NULL)) { - _global_task_pool = gst_task_pool_new (); + if (_global_task_pool) { + if (G_TYPE_FROM_INSTANCE (_global_task_pool) != klass->pool_type) { + GST_INFO ("task pool type changed. Recreating global pool"); + gst_task_pool_cleanup (_global_task_pool); + gst_object_unref (_global_task_pool); + _global_task_pool = NULL; + } + } + + if (_global_task_pool == NULL) { + _global_task_pool = g_object_new (klass->pool_type, NULL); gst_task_pool_prepare (_global_task_pool, NULL); /* Classes are never destroyed so this ref will never be dropped */ @@ -248,6 +257,17 @@ gst_task_class_init (GstTaskClass * klass) gobject_class = (GObjectClass *) klass; gobject_class->finalize = gst_task_finalize; + + klass->pool_type = GST_TYPE_TASK_POOL; +} + +void +gst_task_class_set_default_task_pool_type (GType type) +{ + GstTaskClass *klass = g_type_class_peek (gst_task_get_type ()); + g_mutex_lock (&pool_lock); + klass->pool_type = type; + g_mutex_unlock (&pool_lock); } static void diff --git a/subprojects/gstreamer/gst/gsttask.h b/subprojects/gstreamer/gst/gsttask.h index 7ae1d43abbb..9a7415ad033 100644 --- a/subprojects/gstreamer/gst/gsttask.h +++ b/subprojects/gstreamer/gst/gsttask.h @@ -159,6 +159,7 @@ struct _GstTaskClass { /*< private >*/ GstTaskPool *pool; + GType pool_type; /*< private >*/ gpointer _gst_reserved[GST_PADDING]; @@ -213,6 +214,9 @@ gboolean gst_task_resume (GstTask *task); GST_API gboolean gst_task_join (GstTask *task); +GST_EXPORT +void gst_task_class_set_default_task_pool_type (GType type); + G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstTask, gst_object_unref) G_END_DECLS diff --git a/subprojects/gstreamer/tests/check/gst/struct_x86_64.h b/subprojects/gstreamer/tests/check/gst/struct_x86_64.h index 2708df01cc7..20cfca55684 100644 --- a/subprojects/gstreamer/tests/check/gst/struct_x86_64.h +++ b/subprojects/gstreamer/tests/check/gst/struct_x86_64.h @@ -55,7 +55,7 @@ static GstCheckABIStruct list[] = { {"GstSystemClockClass", sizeof (GstSystemClockClass), 296}, {"GstSystemClock", sizeof (GstSystemClock), 168}, {"GstTagList", sizeof (GstTagList), 64,}, - {"GstTaskClass", sizeof (GstTaskClass), 224}, + {"GstTaskClass", sizeof (GstTaskClass), 232}, {"GstTask", sizeof (GstTask), 200}, {"GstTaskPoolClass", sizeof (GstTaskPoolClass), 248}, {"GstTaskPool", sizeof (GstTaskPool), 128}, diff --git a/subprojects/gstreamer/tests/check/gst/struct_x86_64w.h b/subprojects/gstreamer/tests/check/gst/struct_x86_64w.h index 00c640589e1..51b35d2a135 100644 --- a/subprojects/gstreamer/tests/check/gst/struct_x86_64w.h +++ b/subprojects/gstreamer/tests/check/gst/struct_x86_64w.h @@ -55,7 +55,7 @@ static GstCheckABIStruct list[] = { {"GstSystemClockClass", sizeof (GstSystemClockClass), 296}, {"GstSystemClock", sizeof (GstSystemClock), 168}, {"GstTagList", sizeof (GstTagList), 64,}, - {"GstTaskClass", sizeof (GstTaskClass), 224}, + {"GstTaskClass", sizeof (GstTaskClass), 232}, {"GstTask", sizeof (GstTask), 200}, {"GstTaskPoolClass", sizeof (GstTaskPoolClass), 248}, {"GstTaskPool", sizeof (GstTaskPool), 128}, From 550c2085faf96f96d723bc6c35f1f4c50fd57b4c Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Tue, 25 Mar 2014 14:13:49 +0100 Subject: [PATCH 089/339] paramspecs: Add int range. https://bugzilla.gnome.org/show_bug.cgi?id=773090 --- subprojects/gstreamer/gst/gstparamspecs.c | 145 ++++++++++++++++++++++ subprojects/gstreamer/gst/gstparamspecs.h | 40 ++++++ 2 files changed, 185 insertions(+) diff --git a/subprojects/gstreamer/gst/gstparamspecs.c b/subprojects/gstreamer/gst/gstparamspecs.c index 39c378ce411..f8545adc9eb 100644 --- a/subprojects/gstreamer/gst/gstparamspecs.c +++ b/subprojects/gstreamer/gst/gstparamspecs.c @@ -1,5 +1,6 @@ /* GStreamer - GParamSpecs for some of our types * Copyright (C) 2007 Tim-Philipp Müller + * Copyright (C) 2014 Haakon Sporsheim * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -367,3 +368,147 @@ gst_param_spec_array (const gchar * name, return G_PARAM_SPEC (aspec); } + +/* --- GstParamSpecIntRange --- */ + +static void +_gst_param_int_range_init (GParamSpec * pspec) +{ + GstParamSpecIntRange *irspec = GST_PARAM_SPEC_INT_RANGE (pspec); + + irspec->min_min = G_MININT; + irspec->min_max = G_MININT; + irspec->min_step = G_MININT; + irspec->max_min = G_MAXINT; + irspec->max_max = G_MAXINT; + irspec->max_step = G_MAXINT; + + irspec->def_min = G_MININT; + irspec->def_max = G_MAXINT; + irspec->def_step = 1; +} + +static void +_gst_param_int_range_set_default (GParamSpec * pspec, GValue * value) +{ + GstParamSpecIntRange *irspec = GST_PARAM_SPEC_INT_RANGE (pspec); + + gst_value_set_int_range_step (value, + irspec->def_min, irspec->def_max, irspec->def_step); +} + +static gboolean +_gst_param_int_range_validate (GParamSpec * pspec, GValue * value) +{ + (void)pspec; + + g_return_val_if_fail (GST_VALUE_HOLDS_INT_RANGE (value), TRUE); + g_return_val_if_fail (gst_value_get_int_range_step (value) > 0, TRUE); + + if (gst_value_get_int_range_max (value) < gst_value_get_int_range_min (value)) { + gst_value_set_int_range (value, gst_value_get_int_range_min (value), + gst_value_get_int_range_min (value)); + return TRUE; + } + + return FALSE; +} + +static gint +_gst_param_int_range_values_cmp (GParamSpec * pspec, const GValue * value1, + const GValue * value2) +{ + /* GST_VALUE_LESS_THAN is -1, EQUAL is 0, and GREATER_THAN is 1 */ + return gst_value_compare (value1, value2); +} + +GType +gst_param_spec_int_range_get_type (void) +{ + static GType type; /* 0 */ + + /* register GST_TYPE_PARAM_INT_RANGE */ + if (type == 0) { + static GParamSpecTypeInfo pspec_info = { + sizeof (GstParamSpecIntRange), /* instance_size */ + 0, /* n_preallocs */ + _gst_param_int_range_init, /* instance_init */ + G_TYPE_INVALID, /* value_type */ + NULL, /* finalize */ + _gst_param_int_range_set_default, /* value_set_default */ + _gst_param_int_range_validate, /* value_validate */ + _gst_param_int_range_values_cmp, /* values_cmp */ + }; + pspec_info.value_type = GST_TYPE_INT_RANGE; + type = g_param_type_register_static ("GstParamIntRange", &pspec_info); + } + return type; +} + +/** + * gst_param_spec_int_range: + * @name: canonical name of the property specified + * @nick: nick name for the property specified + * @blurb: description of the property specified + * @min_min: minimum min value + * @min_max: minimum max value + * @min_step: minimum step + * @max_min: maximum min value + * @max_max: maximum max value + * @max_step: maximum step + * @def_min: default min value + * @def_max: default max value + * @def_step: default step + * @flags: flags for the property specified + * + * This function creates an int range GParamSpec for use by objects/elements + * that want to expose properties of int range type. This function is typically + * used in connection with g_object_class_install_property() in a GObjects's + * instance_init function. + * + * Returns: (transfer full): a newly created parameter specification + */ +GParamSpec * +gst_param_spec_int_range (const gchar * name, const gchar * nick, const gchar * blurb, + gint min_min, gint min_max, gint min_step, + gint max_min, gint max_max, gint max_step, + gint def_min, gint def_max, gint def_step, + GParamFlags flags) +{ + GstParamSpecIntRange *irspec; + GParamSpec *pspec; + GValue default_val = G_VALUE_INIT; + + irspec = + g_param_spec_internal (GST_TYPE_PARAM_INT_RANGE, name, nick, blurb, flags); + + irspec->min_min = min_min; + irspec->min_max = min_max; + irspec->min_step = min_step; + irspec->max_min = max_min; + irspec->max_max = max_max; + irspec->max_step = max_step; + irspec->def_min = def_min; + irspec->def_max = def_max; + irspec->def_step = def_step; + + pspec = G_PARAM_SPEC (irspec); + + /* check that min <= default <= max */ + g_value_init (&default_val, GST_TYPE_INT_RANGE); + gst_value_set_int_range_step (&default_val, def_min, def_max, def_step); + + if (_gst_param_int_range_validate (pspec, &default_val)) { + g_critical ("GstParamSpec of type 'int_range' for property '%s' has a " + "default value of %d->%d (%d), which is not within the allowed range of " + "%d->%d to %d->%d", + name, def_min, def_max, def_step, min_min, max_min, min_max, max_max); + g_param_spec_ref (pspec); + g_param_spec_sink (pspec); + g_param_spec_unref (pspec); + pspec = NULL; + } + g_value_unset (&default_val); + + return pspec; +} diff --git a/subprojects/gstreamer/gst/gstparamspecs.h b/subprojects/gstreamer/gst/gstparamspecs.h index ff35507c737..672dbdb1e15 100644 --- a/subprojects/gstreamer/gst/gstparamspecs.h +++ b/subprojects/gstreamer/gst/gstparamspecs.h @@ -117,6 +117,9 @@ G_BEGIN_DECLS #define GST_IS_PARAM_SPEC_ARRAY_LIST(pspec) (G_TYPE_CHECK_INSTANCE_TYPE ((pspec), GST_TYPE_PARAM_ARRAY_LIST)) #define GST_PARAM_SPEC_ARRAY_LIST(pspec) (G_TYPE_CHECK_INSTANCE_CAST ((pspec), GST_TYPE_PARAM_ARRAY_LIST, GstParamSpecArray)) +#define GST_TYPE_PARAM_INT_RANGE (gst_param_spec_int_range_get_type ()) +#define GST_IS_PARAM_SPEC_INT_RANGE(pspec) (G_TYPE_CHECK_INSTANCE_TYPE ((pspec), GST_TYPE_PARAM_INT_RANGE)) +#define GST_PARAM_SPEC_INT_RANGE(pspec) (G_TYPE_CHECK_INSTANCE_CAST ((pspec), GST_TYPE_PARAM_INT_RANGE, GstParamSpecIntRange)) /* --- get_type functions --- */ @@ -126,11 +129,15 @@ GType gst_param_spec_fraction_get_type (void); GST_API GType gst_param_spec_array_get_type (void); +GST_API +GType gst_param_spec_int_range_get_type (void); + /* --- typedefs & structures --- */ typedef struct _GstParamSpecFraction GstParamSpecFraction; typedef struct _GstParamSpecArray GstParamSpecArray; +typedef struct _GstParamSpecIntRange GstParamSpecIntRange; /** * GstParamSpecFraction: @@ -166,6 +173,29 @@ struct _GstParamSpecArray { GParamSpec * element_spec; }; +/** + * GstParamSpecIntRange: + * @parent_instance: super class + * @min_min: minimal lowest value in range + * @min_max: minimal hightst value in range + * @min_step: minimal step + * @max_min: maximal lowest value in range + * @max_max: maximal hightst value in range + * @max_step: maximal step + * @def_min: default lowest value in range + * @def_max: default hightst value in range + * @def_step: default step + * + * A GParamSpec derived structure that contains the meta data for int range + * properties. + */ +struct _GstParamSpecIntRange { + GParamSpec parent_instance; + + gint min_min, min_max, min_step; + gint max_min, max_max, max_step; + gint def_min, def_max, def_step; +}; /* --- GParamSpec prototypes --- */ @@ -184,6 +214,16 @@ GParamSpec * gst_param_spec_array (const gchar * name, GParamSpec * element_spec, GParamFlags flags) G_GNUC_MALLOC; +GST_API +GParamSpec * gst_param_spec_int_range (const gchar * name, + const gchar * nick, + const gchar * blurb, + gint min_min, gint min_max, gint min_step, + gint max_min, gint max_max, gint max_step, + gint def_min, gint def_max, gint def_step, + GParamFlags flags) G_GNUC_MALLOC; + + G_END_DECLS #endif /* __GST_PARAMSPECS_H__ */ From 5634f9fdbb4250de7befd3147791fad5dcda2eda Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Thu, 26 Feb 2015 14:35:35 +0100 Subject: [PATCH 090/339] gstinfo: Make GST_DEBUG_PAD_NAME "MT crash safe" (for gcc) The pad may be unparented while this macro is called which could result in a segfault. The new macro protects against this. The parent may still be disposed while the macro is called, but this will not result in a crash (but the parent name may be garbage). Using gst_pad_get_parent () is undesirable since it takes the object lock. The patch take advantage of compound expressions available as a C extension in GCC and some other compilers. https://bugzilla.gnome.org/show_bug.cgi?id=761916 --- subprojects/gstreamer/gst/gstinfo.h | 25 ++++++- .../gstreamer/tests/check/gst/gstinfo.c | 74 +++++++++++++++++++ 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/subprojects/gstreamer/gst/gstinfo.h b/subprojects/gstreamer/gst/gstinfo.h index 416bd7c25f3..53dccfa541e 100644 --- a/subprojects/gstreamer/gst/gstinfo.h +++ b/subprojects/gstreamer/gst/gstinfo.h @@ -239,6 +239,26 @@ struct _GstDebugCategory { */ #define GST_STR_NULL(str) ((str) ? (str) : "(NULL)") +#if defined (__GNUC__) +#define _GST_OBJECT_PARENT_NAME(obj) \ + ({ \ + GstObject * parent; \ + const gchar * parent_name; \ + if (G_LIKELY ((parent = GST_OBJECT_PARENT (obj)) != NULL)) { \ + /* parent may be disposed, but accessing name will not crash*/ \ + parent_name = GST_OBJECT_NAME (parent); \ + } else { \ + parent_name = "''"; \ + } \ + parent_name; \ + }) +#else +/* Not MT safe since object may be unparented after check */ +#define _GST_OBJECT_PARENT_NAME(obj) \ + (GST_OBJECT_PARENT (obj) != NULL) ? \ + GST_OBJECT_NAME (GST_OBJECT_PARENT (obj)) : "''" +#endif + /* FIXME, not MT safe */ /** * GST_DEBUG_PAD_NAME: @@ -248,10 +268,7 @@ struct _GstDebugCategory { * statements. */ #define GST_DEBUG_PAD_NAME(pad) \ - (pad != NULL) ? \ - ((GST_OBJECT_PARENT(pad) != NULL) ? \ - GST_STR_NULL (GST_OBJECT_NAME (GST_OBJECT_PARENT(pad))) : \ - "''" ) : "''", \ + (pad != NULL) ? GST_STR_NULL (_GST_OBJECT_PARENT_NAME (pad)) : "''", \ (pad != NULL) ? GST_STR_NULL (GST_OBJECT_NAME (pad)) : "''" /** diff --git a/subprojects/gstreamer/tests/check/gst/gstinfo.c b/subprojects/gstreamer/tests/check/gst/gstinfo.c index 8615e4802a0..43120ec8027 100644 --- a/subprojects/gstreamer/tests/check/gst/gstinfo.c +++ b/subprojects/gstreamer/tests/check/gst/gstinfo.c @@ -558,6 +558,74 @@ GST_START_TEST (info_set_and_reset_string) GST_END_TEST; + +typedef struct +{ + gboolean thread_running; + gpointer user_data; +} StressTestData; + +static gpointer +debug_pad_name_func (gpointer user_data) +{ + StressTestData *data = user_data; + GstPad *pad = data->user_data; + + while (data->thread_running) { + gchar *name = g_strdup_printf ("%s:%s", GST_DEBUG_PAD_NAME (pad)); + fail_unless (name != NULL); + g_free (name); + } + + return NULL; +} + +static gpointer +unparent_func (gpointer user_data) +{ + StressTestData *data = user_data; + GstObject *obj = data->user_data; + + while (data->thread_running) { + GstObject *parent = gst_object_get_parent (obj); + gst_object_unparent (obj); + g_thread_yield (); + gst_object_set_parent (obj, parent); + gst_object_unref (parent); + } + + return NULL; +} + +GST_START_TEST(gst_debug_pad_name_stress) +{ + GThread *pad_name_thread, *unparent_thread; + GstPad *pad = gst_pad_new ("testpad", GST_PAD_UNKNOWN); + GstElement *element = gst_element_factory_make ("identity", NULL); + StressTestData data = { TRUE, pad }; + + g_object_ref (G_OBJECT (pad)); + gst_object_set_parent (GST_OBJECT (pad), GST_OBJECT (element)); + + pad_name_thread = g_thread_new ( + "pad_name_thread", (GThreadFunc) debug_pad_name_func, &data); + unparent_thread = g_thread_new ( + "gst_object_unparent_thread", (GThreadFunc) unparent_func, &data); + + /* test duration */ + g_usleep (G_USEC_PER_SEC / 10); + + data.thread_running = FALSE; + g_thread_join (pad_name_thread); + g_thread_join (unparent_thread); + + gst_object_unparent (GST_OBJECT (pad)); + gst_object_unref (pad); + gst_object_unref (element); +} + +GST_END_TEST; + static Suite * gst_info_suite (void) { @@ -583,6 +651,12 @@ gst_info_suite (void) tcase_add_test (tc_chain, info_set_and_reset_string); #endif +#if defined (__GNUC__) + tcase_add_test (tc_chain, gst_debug_pad_name_stress); +#else + tcase_skip_broken_test (tc_chain, gst_debug_pad_name_stress); +#endif + return s; } From e4c83d1f744a39646a408fcd93dbf687a65d8b81 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 11 Mar 2016 15:24:55 +0100 Subject: [PATCH 091/339] basetransform: Fix race when toggling passthrough This fixes a race where priv->passthrough is changed from TRUE to FALSE while processing a buffer and we end up passing a non-writable buffer to transform_ip(). More precicely if passthrough is changed just after prepare_output_buffer() is finished. Since priv->passthrough and other priv variables are accessed throughout the chain function a lock is introduced and held while processing the buffer, but released before pushing downstream. Since sub-classes may call is_passthrough() and similar functions during for instance transform_ip() a recursive lock is needed. https://bugzilla.gnome.org/show_bug.cgi?id=773093 --- .../libs/gst/base/gstbasetransform.c | 91 ++++++---- .../tests/check/libs/test_transform.c | 34 ++-- .../gstreamer/tests/check/libs/transform1.c | 168 ++++++++++++++++++ 3 files changed, 244 insertions(+), 49 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/base/gstbasetransform.c b/subprojects/gstreamer/libs/gst/base/gstbasetransform.c index 3f38c094b20..0434e98d066 100644 --- a/subprojects/gstreamer/libs/gst/base/gstbasetransform.c +++ b/subprojects/gstreamer/libs/gst/base/gstbasetransform.c @@ -163,12 +163,17 @@ enum PROP_QOS }; +#define PRIV_LOCK(trans) g_rec_mutex_lock (&(trans)->priv->lock) +#define PRIV_UNLOCK(trans) g_rec_mutex_unlock (&(trans)->priv->lock) + struct _GstBaseTransformPrivate { /* Set by sub-class */ gboolean passthrough; gboolean always_in_place; + GRecMutex lock; + GstCaps *cache_caps1; gsize cache_caps1_size; GstCaps *cache_caps2; @@ -314,6 +319,10 @@ gst_base_transform_default_transform_meta (GstBaseTransform * trans, static void gst_base_transform_finalize (GObject * object) { + GstBaseTransform *trans = GST_BASE_TRANSFORM_CAST (object); + + g_rec_mutex_clear (&trans->priv->lock); + G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -381,6 +390,7 @@ gst_base_transform_init (GstBaseTransform * trans, GST_DEBUG ("gst_base_transform_init"); priv = trans->priv = gst_base_transform_get_instance_private (trans); + g_rec_mutex_init (&priv->lock); pad_template = gst_element_class_get_pad_template (GST_ELEMENT_CLASS (bclass), "sink"); @@ -753,7 +763,7 @@ gst_base_transform_set_allocation (GstBaseTransform * trans, GstQuery *oldquery; GstBaseTransformPrivate *priv = trans->priv; - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); oldpool = priv->pool; priv->pool = pool; priv->pool_active = FALSE; @@ -768,7 +778,7 @@ gst_base_transform_set_allocation (GstBaseTransform * trans, priv->params = *params; else gst_allocation_params_init (&priv->params); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); if (oldpool) { GST_DEBUG_OBJECT (trans, "deactivating old pool %p", oldpool); @@ -1359,9 +1369,9 @@ gst_base_transform_setcaps (GstBaseTransform * trans, GstPad * pad, if (prev_outcaps) gst_caps_unref (prev_outcaps); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); priv->negotiated = ret; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); return ret; @@ -1516,17 +1526,17 @@ gst_base_transform_default_query (GstBaseTransform * trans, if (G_UNLIKELY (!ret)) goto done; - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (!priv->negotiated && !priv->passthrough && (klass->set_caps != NULL)) { GST_DEBUG_OBJECT (trans, "not negotiated yet but need negotiation, can't answer ALLOCATION query"); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); goto done; } decide_query = trans->priv->query; trans->priv->query = NULL; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); GST_DEBUG_OBJECT (trans, "calling propose allocation with query %" GST_PTR_FORMAT, @@ -1539,14 +1549,14 @@ gst_base_transform_default_query (GstBaseTransform * trans, ret = FALSE; if (decide_query) { - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (trans->priv->query == NULL) trans->priv->query = decide_query; else gst_query_unref (decide_query); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } GST_DEBUG_OBJECT (trans, "ALLOCATION ret %d, %" GST_PTR_FORMAT, ret, @@ -1924,14 +1934,14 @@ gst_base_transform_sink_eventfunc (GstBaseTransform * trans, GstEvent * event) case GST_EVENT_FLUSH_START: break; case GST_EVENT_FLUSH_STOP: - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); /* reset QoS parameters */ priv->proportion = 1.0; priv->earliest_time = -1; priv->discont = FALSE; priv->processed = 0; priv->dropped = 0; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); /* we need new segment info after the flush. */ trans->have_segment = FALSE; gst_segment_init (&trans->segment, GST_FORMAT_UNDEFINED); @@ -2056,7 +2066,8 @@ default_submit_input_buffer (GstBaseTransform * trans, gboolean is_discont, * or if the class doesn't implement a set_caps function (in which case it doesn't * care about caps) */ - if (!priv->negotiated && !priv->passthrough && (bclass->set_caps != NULL)) + if (!priv->negotiated && !priv->passthrough && + (bclass->set_caps != NULL)) goto not_negotiated; GST_OBJECT_LOCK (trans); @@ -2083,13 +2094,14 @@ default_submit_input_buffer (GstBaseTransform * trans, gboolean is_discont, /* lock for getting the QoS parameters that are set (in a different thread) * with the QOS events */ - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); earliest_time = priv->earliest_time; proportion = priv->proportion; /* check for QoS, don't perform conversion for buffers * that are known to be late. */ - need_skip = earliest_time != -1 && running_time <= earliest_time; - GST_OBJECT_UNLOCK (trans); + need_skip = priv->qos_enabled && + earliest_time != -1 && running_time <= earliest_time; + PRIV_UNLOCK (trans); if (need_skip) { GstMessage *qos_msg; @@ -2324,6 +2336,8 @@ gst_base_transform_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GstClockTime timestamp, duration; GstBuffer *outbuf = NULL; + PRIV_LOCK (trans); + timestamp = GST_BUFFER_TIMESTAMP (buffer); duration = GST_BUFFER_DURATION (buffer); @@ -2345,7 +2359,9 @@ gst_base_transform_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) } /* Takes ownership of input buffer */ + PRIV_UNLOCK (trans); ret = klass->submit_input_buffer (trans, priv->discont, buffer); + PRIV_LOCK (trans); if (ret != GST_FLOW_OK) goto done; @@ -2354,6 +2370,8 @@ gst_base_transform_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) ret = klass->generate_output (trans, &outbuf); + PRIV_UNLOCK (trans); + /* outbuf can be NULL, this means a dropped buffer, if we have a buffer but * GST_BASE_TRANSFORM_FLOW_DROPPED we will not push either. */ if (outbuf != NULL) { @@ -2394,9 +2412,12 @@ gst_base_transform_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) gst_buffer_unref (outbuf); } } + PRIV_LOCK (trans); } while (ret == GST_FLOW_OK && outbuf != NULL); done: + PRIV_UNLOCK (trans); + /* convert internal flow to OK and mark discont for the next buffer. */ if (ret == GST_BASE_TRANSFORM_FLOW_DROPPED) { GST_DEBUG_OBJECT (trans, "dropped a buffer, marking DISCONT"); @@ -2462,7 +2483,7 @@ gst_base_transform_activate (GstBaseTransform * trans, gboolean active) incaps = gst_pad_get_current_caps (trans->sinkpad); outcaps = gst_pad_get_current_caps (trans->srcpad); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (incaps && outcaps) priv->have_same_caps = gst_caps_is_equal (incaps, outcaps) || priv->passthrough; @@ -2478,7 +2499,7 @@ gst_base_transform_activate (GstBaseTransform * trans, gboolean active) priv->discont = FALSE; priv->processed = 0; priv->dropped = 0; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); if (incaps) gst_caps_unref (incaps); @@ -2590,7 +2611,7 @@ gst_base_transform_set_passthrough (GstBaseTransform * trans, bclass = GST_BASE_TRANSFORM_GET_CLASS (trans); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (!passthrough) { if (bclass->transform_ip || bclass->transform || (bclass->generate_output && bclass->generate_output != default_generate_output)) @@ -2600,7 +2621,7 @@ gst_base_transform_set_passthrough (GstBaseTransform * trans, } GST_DEBUG_OBJECT (trans, "set passthrough %d", trans->priv->passthrough); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2620,9 +2641,9 @@ gst_base_transform_is_passthrough (GstBaseTransform * trans) g_return_val_if_fail (GST_IS_BASE_TRANSFORM (trans), FALSE); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); result = trans->priv->passthrough; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); return result; } @@ -2650,7 +2671,7 @@ gst_base_transform_set_in_place (GstBaseTransform * trans, gboolean in_place) bclass = GST_BASE_TRANSFORM_GET_CLASS (trans); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (in_place) { if (bclass->transform_ip) { @@ -2664,7 +2685,7 @@ gst_base_transform_set_in_place (GstBaseTransform * trans, gboolean in_place) } } - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2684,9 +2705,9 @@ gst_base_transform_is_in_place (GstBaseTransform * trans) g_return_val_if_fail (GST_IS_BASE_TRANSFORM (trans), FALSE); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); result = trans->priv->always_in_place; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); return result; } @@ -2716,10 +2737,10 @@ gst_base_transform_update_qos (GstBaseTransform * trans, "qos: proportion: %lf, diff %" G_GINT64_FORMAT ", timestamp %" GST_TIME_FORMAT, proportion, diff, GST_TIME_ARGS (timestamp)); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); trans->priv->proportion = proportion; trans->priv->earliest_time = timestamp + diff; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2738,9 +2759,9 @@ gst_base_transform_set_qos_enabled (GstBaseTransform * trans, gboolean enabled) GST_CAT_DEBUG_OBJECT (GST_CAT_QOS, trans, "enabled: %d", enabled); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); trans->priv->qos_enabled = enabled; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2760,9 +2781,9 @@ gst_base_transform_is_qos_enabled (GstBaseTransform * trans) g_return_val_if_fail (GST_IS_BASE_TRANSFORM (trans), FALSE); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); result = trans->priv->qos_enabled; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); return result; } @@ -2786,10 +2807,10 @@ gst_base_transform_set_gap_aware (GstBaseTransform * trans, gboolean gap_aware) { g_return_if_fail (GST_IS_BASE_TRANSFORM (trans)); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); trans->priv->gap_aware = gap_aware; GST_DEBUG_OBJECT (trans, "set gap aware %d", trans->priv->gap_aware); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2817,10 +2838,10 @@ gst_base_transform_set_prefer_passthrough (GstBaseTransform * trans, { g_return_if_fail (GST_IS_BASE_TRANSFORM (trans)); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); trans->priv->prefer_passthrough = prefer_passthrough; GST_DEBUG_OBJECT (trans, "prefer passthrough %d", prefer_passthrough); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** diff --git a/subprojects/gstreamer/tests/check/libs/test_transform.c b/subprojects/gstreamer/tests/check/libs/test_transform.c index 534599b6c18..d380f9d4aba 100644 --- a/subprojects/gstreamer/tests/check/libs/test_transform.c +++ b/subprojects/gstreamer/tests/check/libs/test_transform.c @@ -165,30 +165,36 @@ result_buffer_alloc (GstPad * pad, guint64 offset, guint size, GstCaps * caps, } #endif -static TestTransData * -gst_test_trans_new (void) +static GstElement * +gst_test_trans_element_new (void) { - TestTransData *res; - GstPad *tmp; - GstPadTemplate *templ; GType type; /* we register a new sub-class for every test-run, so the class init * function is called for every test run and can be set up properly * even with CK_FORK=no */ - { - static gint counter = 0; - gchar name[100]; + static gint counter = 0; + gchar name[100]; - g_snprintf (name, sizeof (name), "GstTestTrans%d", ++counter); + g_snprintf (name, sizeof (name), "GstTestTrans%d", ++counter); - type = g_type_register_static_simple (GST_TYPE_BASE_TRANSFORM, name, - sizeof (GstTestTransClass), (GClassInitFunc) gst_test_trans_class_init, - sizeof (GstTestTrans), (GInstanceInitFunc) gst_test_trans_init, 0); - } + type = g_type_register_static_simple (GST_TYPE_BASE_TRANSFORM, name, + sizeof (GstTestTransClass), (GClassInitFunc) gst_test_trans_class_init, + sizeof (GstTestTrans), (GInstanceInitFunc) gst_test_trans_init, 0); + + return g_object_new (type, NULL); +} + + +static TestTransData * +gst_test_trans_new (void) +{ + TestTransData *res; + GstPad *tmp; + GstPadTemplate *templ; res = g_new0 (TestTransData, 1); - res->trans = g_object_new (type, NULL); + res->trans = gst_test_trans_element_new (); templ = gst_static_pad_template_get (sink_template); templ->direction = GST_PAD_SRC; diff --git a/subprojects/gstreamer/tests/check/libs/transform1.c b/subprojects/gstreamer/tests/check/libs/transform1.c index 2b9cb1e17eb..2dcd6f0f705 100644 --- a/subprojects/gstreamer/tests/check/libs/transform1.c +++ b/subprojects/gstreamer/tests/check/libs/transform1.c @@ -25,6 +25,7 @@ #endif #include #include +#include #include #include "test_transform.c" @@ -941,6 +942,170 @@ GST_START_TEST (basetransform_invalid_fixatecaps_impl) GST_END_TEST; +static GstFlowReturn +transform_ip_assert_writable (GstBaseTransform * trans, GstBuffer * buf) +{ + GST_DEBUG_OBJECT (trans, "transform called"); + + if (!gst_base_transform_is_passthrough (trans)) + fail_unless (gst_buffer_is_writable (buf)); + + return GST_FLOW_OK; +} + +static void +toggle_passthrough (gpointer data, gpointer user_data) +{ + GstBaseTransform *basetrans = GST_BASE_TRANSFORM (user_data); + + gst_base_transform_set_passthrough (basetrans, TRUE); + g_thread_yield (); + gst_base_transform_set_passthrough (basetrans, FALSE); +} + +GST_START_TEST (basetransform_stress_pt_ip) +{ + GstHarness *h[10]; + GstHarnessThread *push[10], *pt[10]; + GstElement *trans; + GstBuffer *buffer; + GstSegment segment; + GstCaps *caps; + guint i; + + klass_transform_ip = transform_ip_assert_writable; + gst_segment_init (&segment, GST_FORMAT_TIME); + caps = gst_caps_from_string ("foo/x-bar"); + buffer = gst_buffer_new (); + + for (i = 0; i < G_N_ELEMENTS (h); i++) { + trans = gst_test_trans_element_new (); + h[i] = gst_harness_new_with_element (trans, "sink", "src"); + push[i] = gst_harness_stress_push_buffer_start (h[i], caps, &segment, buffer); + pt[i] = gst_harness_stress_custom_start (h[i], NULL, toggle_passthrough, trans, 0); + g_object_unref (trans); + } + gst_caps_unref (caps); + gst_buffer_unref (buffer); + + g_usleep (G_USEC_PER_SEC/10); + + for (i = 0; i < G_N_ELEMENTS(h); i++) { + gst_harness_stress_thread_stop (pt[i]); + gst_harness_stress_thread_stop (push[i]); + gst_harness_teardown (h[i]); + } +} + +GST_END_TEST; + +static gboolean +test_harness_sink_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstPadQueryFunction query_func = g_object_get_qdata (G_OBJECT (pad), + g_quark_from_static_string ("test-data-harness-query-func")); + + if (GST_QUERY_TYPE (query) == GST_QUERY_ALLOCATION) { + GstPromise **p = g_object_get_qdata (G_OBJECT (pad), + g_quark_from_static_string ("test-data-promises")); + + GstPromise *padblocked = p[0]; + GstPromise *padunblocked = p[1]; + // Notifay main thread that basetransform sent allocation query + // and waite while the main thread changes the 'passthrough' flag + gst_promise_reply (padblocked, NULL); + gst_promise_wait (padunblocked); + } + return query_func (pad, parent, query); +} + +static GstFlowReturn +transform_pt_ct_alloc_query (GstBaseTransform * trans, + GstBuffer * in, GstBuffer * out) +{ + return GST_FLOW_OK; +} + +static gboolean +reconfigure_each_pad (GstElement * element, + GstPad * pad, + gpointer user_data) +{ + (void) user_data; + gst_pad_mark_reconfigure (pad); + return TRUE; +} + +static GstBuffer * +before_buffer_push (GstHarness * h, gpointer data) +{ + gst_element_foreach_src_pad(h->element, reconfigure_each_pad, NULL); + gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (h->element), FALSE); + return gst_buffer_ref (GST_BUFFER_CAST (data)); +} + +/* Test for capability to modification/read basetransform privat data + * while it is wating for query results (in this case allocation query). +*/ +GST_START_TEST (basetransform_stress_pt_ct_alloc_query) +{ + GstHarness *h = NULL; + GstHarnessThread *push = NULL; + GstElement *trans = NULL; + GstBuffer *buffer = NULL; + GstSegment segment; + GstCaps *caps = NULL; + GstPromise *padblocked = gst_promise_new (); + GstPromise *padunblocked = gst_promise_new (); + GstPromise *p[2] = {padblocked, padunblocked}; + GstPadQueryFunction old_query_func = NULL; + + klass_transform = transform_pt_ct_alloc_query; + gst_segment_init (&segment, GST_FORMAT_TIME); + caps = gst_caps_from_string ("foo/x-bar"); + buffer = gst_buffer_new (); + + trans = gst_test_trans_element_new (); + h = gst_harness_new_with_element (trans, "sink", "src"); + + old_query_func = GST_PAD_QUERYFUNC (h->sinkpad); + // Store the old harness query handler + g_object_set_qdata (G_OBJECT (h->sinkpad), + g_quark_from_static_string ("test-data-harness-query-func"), + old_query_func); + + g_object_set_qdata (G_OBJECT (h->sinkpad), + g_quark_from_static_string ("test-data-promises"), p); + + // Set the new harness query handler to intercect the allocation + // queries. + gst_pad_set_query_function (h->sinkpad, test_harness_sink_query); + + // Start the dedicated push buffer thread. + push = gst_harness_stress_push_buffer_with_cb_start(h, caps, &segment, + before_buffer_push, buffer, (GDestroyNotify) gst_buffer_unref); + + // Wait for notification from harness pad when basetransform sents allocation query. + gst_promise_wait (padblocked); + // Now we should be able to modifay the basetransform passthrough mode. + gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (h->element), TRUE); + // Notifay harness thread to continue buffer processing + gst_promise_reply (padunblocked, NULL); + + gst_harness_stress_thread_stop (push); + + GST_PAD_QUERYFUNC (h->sinkpad) = old_query_func; + + gst_buffer_unref (buffer); + gst_caps_unref (caps); + g_object_unref (trans); + gst_harness_teardown (h); + gst_promise_unref (padblocked); + gst_promise_unref (padunblocked); +} + +GST_END_TEST; + static void transform1_setup (void) { @@ -981,6 +1146,9 @@ gst_basetransform_suite (void) tcase_add_test (tc, basetransform_chain_ct1); tcase_add_test (tc, basetransform_chain_ct2); tcase_add_test (tc, basetransform_chain_ct3); + /* stress */ + tcase_add_test (tc, basetransform_stress_pt_ip); + tcase_add_test (tc, basetransform_stress_pt_ct_alloc_query); tcase_add_test (tc, basetransform_invalid_fixatecaps_impl); From 889ece67312291f6f935876ac57a4ac94ee9cddd Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 1 Dec 2015 13:57:17 +1100 Subject: [PATCH 092/339] bin: add disabled test documenting locked-state and base-time problems https://bugzilla.gnome.org/show_bug.cgi?id=773097 --- .../gstreamer/tests/check/gst/gstbin.c | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/subprojects/gstreamer/tests/check/gst/gstbin.c b/subprojects/gstreamer/tests/check/gst/gstbin.c index 88ff44db0c3..0f50d283ecb 100644 --- a/subprojects/gstreamer/tests/check/gst/gstbin.c +++ b/subprojects/gstreamer/tests/check/gst/gstbin.c @@ -25,6 +25,7 @@ #include #include +#include static void pop_async_done (GstBus * bus, gboolean * had_latency) @@ -1988,6 +1989,57 @@ GST_START_TEST (test_suppressed_flags_when_removing) GST_END_TEST; +GST_START_TEST (test_locked_state_sets_base_time_on_children) +{ + GstElement *pipeline, *bin, *identity; + GstClock *testclock; + GstStateChangeReturn state_res; + GstState current, pending; + + /* create the pipeline, and have it use the testclock */ + pipeline = gst_pipeline_new (NULL); + testclock = gst_test_clock_new (); + gst_pipeline_use_clock (GST_PIPELINE (pipeline), testclock); + + /* create a bin, and add an identity element to it */ + bin = gst_bin_new (NULL); + identity = gst_element_factory_make ("identity", NULL); + gst_bin_add (GST_BIN (bin), identity); + + /* add the bin to the pipeline and set locked-state, + protecting the bin from the pipeline state-changes */ + gst_bin_add (GST_BIN (pipeline), bin); + gst_element_set_locked_state (bin, TRUE); + + /* advance the clock, and set to playing, making this our base-time */ + gst_test_clock_set_time (GST_TEST_CLOCK (testclock), 123456789); + state_res = gst_element_set_state (pipeline, GST_STATE_PLAYING); + fail_unless_equals_int (GST_STATE_CHANGE_SUCCESS, state_res); + fail_unless_equals_int64 (123456789, gst_element_get_base_time (pipeline)); + + /* verify that neither the bin nor the identity-element got affected by + the state-change */ + state_res = gst_element_get_state (GST_ELEMENT (bin), + ¤t, &pending, GST_CLOCK_TIME_NONE); + fail_unless_equals_int (GST_STATE_CHANGE_SUCCESS, state_res); + fail_unless_equals_int (GST_STATE_NULL, current); + fail_unless_equals_int (GST_STATE_VOID_PENDING, pending); + + state_res = gst_element_get_state (GST_ELEMENT (identity), + ¤t, &pending, GST_CLOCK_TIME_NONE); + fail_unless_equals_int (GST_STATE_CHANGE_SUCCESS, state_res); + fail_unless_equals_int (GST_STATE_NULL, current); + fail_unless_equals_int (GST_STATE_VOID_PENDING, pending); + + /* and check that base-time got distributed to everyone */ + fail_unless_equals_int64 (123456789, gst_element_get_base_time (bin)); + fail_unless_equals_int64 (123456789, gst_element_get_base_time (identity)); + + gst_object_unref (pipeline); +} + +GST_END_TEST; + static Suite * gst_bin_suite (void) { @@ -2023,6 +2075,7 @@ gst_bin_suite (void) tcase_add_test (tc_chain, test_deep_added_removed); tcase_add_test (tc_chain, test_suppressed_flags); tcase_add_test (tc_chain, test_suppressed_flags_when_removing); + tcase_skip_broken_test (tc_chain, test_locked_state_sets_base_time_on_children); /* fails on OSX build bot for some reason, and is a bit silly anyway */ if (0) From 1b7c26129d45b6659e8363b631fb7a04b9d652b5 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 29 Jan 2016 16:51:56 +0100 Subject: [PATCH 093/339] element: bin: add flag to skip uniqueness checking https://bugzilla.gnome.org/show_bug.cgi?id=773095 --- subprojects/gstreamer/gst/gstbin.c | 5 +++-- subprojects/gstreamer/gst/gstelement.c | 5 +++-- subprojects/gstreamer/gst/gstelement.h | 13 +++++++------ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/subprojects/gstreamer/gst/gstbin.c b/subprojects/gstreamer/gst/gstbin.c index 8c66e35c52a..64afef944c7 100644 --- a/subprojects/gstreamer/gst/gstbin.c +++ b/subprojects/gstreamer/gst/gstbin.c @@ -1183,8 +1183,9 @@ gst_bin_add_func (GstBin * bin, GstElement * element) * we can safely take the lock here. This check is probably bogus because * you can safely change the element name after this check and before setting * the object parent. The window is very small though... */ - if (G_UNLIKELY (!gst_object_check_uniqueness (bin->children, elem_name))) - goto duplicate_name; + if (!GST_OBJECT_FLAG_IS_SET (element, GST_ELEMENT_FLAG_NO_UNIQUE_CHECK)) + if (G_UNLIKELY (!gst_object_check_uniqueness (bin->children, elem_name))) + goto duplicate_name; /* set the element's parent and add the element to the bin's list of children */ if (G_UNLIKELY (!gst_object_set_parent (GST_OBJECT_CAST (element), diff --git a/subprojects/gstreamer/gst/gstelement.c b/subprojects/gstreamer/gst/gstelement.c index 10ce62ba62c..08b2b077be4 100644 --- a/subprojects/gstreamer/gst/gstelement.c +++ b/subprojects/gstreamer/gst/gstelement.c @@ -763,8 +763,9 @@ gst_element_add_pad (GstElement * element, GstPad * pad) /* then check to see if there's already a pad by that name here */ GST_OBJECT_LOCK (element); - if (G_UNLIKELY (!gst_object_check_uniqueness (element->pads, pad_name))) - goto name_exists; + if (!GST_OBJECT_FLAG_IS_SET (element, GST_ELEMENT_FLAG_NO_UNIQUE_CHECK)) + if (G_UNLIKELY (!gst_object_check_uniqueness (element->pads, pad_name))) + goto name_exists; /* try to set the pad's parent */ if (G_UNLIKELY (!gst_object_set_parent (GST_OBJECT_CAST (pad), diff --git a/subprojects/gstreamer/gst/gstelement.h b/subprojects/gstreamer/gst/gstelement.h index e690bf0e3f0..59e03e8f5d4 100644 --- a/subprojects/gstreamer/gst/gstelement.h +++ b/subprojects/gstreamer/gst/gstelement.h @@ -407,12 +407,13 @@ typedef enum /*< flags=0 >*/ */ typedef enum { - GST_ELEMENT_FLAG_LOCKED_STATE = (GST_OBJECT_FLAG_LAST << 0), - GST_ELEMENT_FLAG_SINK = (GST_OBJECT_FLAG_LAST << 1), - GST_ELEMENT_FLAG_SOURCE = (GST_OBJECT_FLAG_LAST << 2), - GST_ELEMENT_FLAG_PROVIDE_CLOCK = (GST_OBJECT_FLAG_LAST << 3), - GST_ELEMENT_FLAG_REQUIRE_CLOCK = (GST_OBJECT_FLAG_LAST << 4), - GST_ELEMENT_FLAG_INDEXABLE = (GST_OBJECT_FLAG_LAST << 5), + GST_ELEMENT_FLAG_LOCKED_STATE = (GST_OBJECT_FLAG_LAST << 0), + GST_ELEMENT_FLAG_SINK = (GST_OBJECT_FLAG_LAST << 1), + GST_ELEMENT_FLAG_SOURCE = (GST_OBJECT_FLAG_LAST << 2), + GST_ELEMENT_FLAG_PROVIDE_CLOCK = (GST_OBJECT_FLAG_LAST << 3), + GST_ELEMENT_FLAG_REQUIRE_CLOCK = (GST_OBJECT_FLAG_LAST << 4), + GST_ELEMENT_FLAG_INDEXABLE = (GST_OBJECT_FLAG_LAST << 5), + GST_ELEMENT_FLAG_NO_UNIQUE_CHECK = (GST_OBJECT_FLAG_LAST << 6), /* padding */ GST_ELEMENT_FLAG_LAST = (GST_OBJECT_FLAG_LAST << 10) } GstElementFlags; From 3e7e3215ed4816bfce91d80d8e7c0d529451b1c2 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 6 May 2016 09:47:32 +0200 Subject: [PATCH 094/339] Add GST_EVENT_LATENCY_CHANGED clipped from #gstreamer: Lets start with some history: Back in 2008 we needed a dynamic jitterbuffer. After discussing back and forth with wtay, we came to the conclusion that the proper way to do this was to: 1. Change the latency property on the jitterbuffer 2. The jb would then post a latency message on the bus 3. in the bus-handler (in the application), this msg would be picked up, and wtay added a new method gst_bin_recalculate_latency that the application could emit on the main pipeline to have the whole pipeline query its latency again (each sink sending upstream latency queries etc). Now this has worked ok for us ever since, in Pexip we terminate the latency in the mixers, not in the sinks, but other then that it works exactly the same way as before. But lately we have discovered that this has some horrific side-effects, in that with many participants connecting, and latency changing often, we in fact, using this method, are recalculating EVERYONES latency when we only want to reconfigure that of a single participant. or more specifically the jitterbuffers with the same cname belonging to that participant. Now, I know the idea of "groups of sinks" have been mentioned before, but we are at a stage now where we really need to be able to re-calculate latency only for the affected sinks / mixers, and this is where I have a concrete suggestions: a downstream "latency-changed" event that in turn can trigger an upstream latency query, either from a sink or from a mixer. If this was emitted from the jitterbuffer in the same place the latency-message is emitted, it would mean all affected downstream mixers/sinks would know "directly" that latency had changed somewhere upstream from them, and could make decisions based on this. For us it would mean re-sending latency-queries upstream from the mixers that received this event, and then using this new latency in the mix. For this sink-case, it would be the same way, only that with using this "mode" the sinks would get their new latency "independent" of the pipeline latency, and the "grouping" would then happen from whichever sources decided to send this event. So in the case of rtpbin, you are already changing latency for all the jitterbuffers with the same cname as one "group", and hence all of them would also be sending the latency event, ensuring lipsync at the mixers/sinks. https://bugzilla.gnome.org/show_bug.cgi?id=764396 --- subprojects/gstreamer/gst/gstevent.c | 25 +++++++++++++++++++++++++ subprojects/gstreamer/gst/gstevent.h | 7 +++++++ subprojects/gstreamer/gst/gstquark.c | 2 +- subprojects/gstreamer/gst/gstquark.h | 3 ++- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/subprojects/gstreamer/gst/gstevent.c b/subprojects/gstreamer/gst/gstevent.c index 5032f1ce9d6..7a2b1640848 100644 --- a/subprojects/gstreamer/gst/gstevent.c +++ b/subprojects/gstreamer/gst/gstevent.c @@ -122,6 +122,7 @@ static GstEventQuarks event_quarks[] = { {GST_EVENT_SEEK, "seek", 0}, {GST_EVENT_NAVIGATION, "navigation", 0}, {GST_EVENT_LATENCY, "latency", 0}, + {GST_EVENT_LATENCY_CHANGED, "latency-changed", 0}, {GST_EVENT_STEP, "step", 0}, {GST_EVENT_RECONFIGURE, "reconfigure", 0}, {GST_EVENT_TOC_SELECT, "toc-select", 0}, @@ -1583,6 +1584,30 @@ gst_event_parse_latency (GstEvent * event, GstClockTime * latency) (event), GST_QUARK (LATENCY))); } +/** + * gst_event_new_latency_changed: + * + * Create a new latency-changed event. The event is sent downstream from + * an element that changes its latency while running, notifying any + * downstream element of this change. + * + * This information is most useful in mixer and sinks, that might send + * an upstream latency-query as a response to figure out the new latency. + * + * Returns: (transfer full): a new #GstEvent + */ +GstEvent * +gst_event_new_latency_changed (void) +{ + GstEvent *event; + + GST_CAT_INFO (GST_CAT_EVENT, "creating latency-changed event"); + + event = gst_event_new_custom (GST_EVENT_LATENCY_CHANGED, NULL); + + return event; +} + /** * gst_event_new_step: * @format: the format of @amount diff --git a/subprojects/gstreamer/gst/gstevent.h b/subprojects/gstreamer/gst/gstevent.h index a58eb40aa82..e7eb75a8604 100644 --- a/subprojects/gstreamer/gst/gstevent.h +++ b/subprojects/gstreamer/gst/gstevent.h @@ -116,6 +116,8 @@ typedef enum { * to upstream elements. * @GST_EVENT_LATENCY: Notification of new latency adjustment. Sinks will use * the latency information to adjust their synchronisation. + * @GST_EVENT_LATENCY_CHANGED: Notification of new latency adjustment upstream. + Sinks can use this to query upstream for the new latency. * @GST_EVENT_STEP: A request for stepping through the media. Sinks will usually * execute the step operation. * @GST_EVENT_RECONFIGURE: A request for upstream renegotiating caps and reconfiguring. @@ -167,6 +169,7 @@ typedef enum { /* non-sticky downstream serialized */ GST_EVENT_SEGMENT_DONE = GST_EVENT_MAKE_TYPE (150, _FLAG(DOWNSTREAM) | _FLAG(SERIALIZED)), GST_EVENT_GAP = GST_EVENT_MAKE_TYPE (160, _FLAG(DOWNSTREAM) | _FLAG(SERIALIZED)), + GST_EVENT_LATENCY_CHANGED = GST_EVENT_MAKE_TYPE (170, _FLAG(DOWNSTREAM) | _FLAG(SERIALIZED)), /* sticky downstream non-serialized */ /* FIXME 2.0: change to value 72 and move after the GST_EVENT_SEGMENT event */ @@ -684,6 +687,10 @@ GstEvent* gst_event_new_latency (GstClockTime latency) G_GNUC_MA GST_API void gst_event_parse_latency (GstEvent *event, GstClockTime *latency); +/* latency-changed event */ +GST_EXPORT +GstEvent* gst_event_new_latency_changed (void) G_GNUC_MALLOC; + /* step event */ GST_API diff --git a/subprojects/gstreamer/gst/gstquark.c b/subprojects/gstreamer/gst/gstquark.c index 304e274155d..3138d9eea44 100644 --- a/subprojects/gstreamer/gst/gstquark.c +++ b/subprojects/gstreamer/gst/gstquark.c @@ -80,7 +80,7 @@ static const gchar *_quark_strings[] = { "GstEventInstantRateChange", "GstEventInstantRateSyncTime", "GstMessageInstantRateRequest", "upstream-running-time", "base", "offset", "plugin-api", "plugin-api-flags", - "gap-flags", "GstQuerySelectable", "selectable" + "gap-flags", "GstQuerySelectable", "selectable", "latency-changed" }; GQuark _priv_gst_quark_table[GST_QUARK_MAX]; diff --git a/subprojects/gstreamer/gst/gstquark.h b/subprojects/gstreamer/gst/gstquark.h index f7de510c9b1..ecb27db6314 100644 --- a/subprojects/gstreamer/gst/gstquark.h +++ b/subprojects/gstreamer/gst/gstquark.h @@ -233,7 +233,8 @@ typedef enum _GstQuarkId GST_QUARK_GAP_FLAGS = 202, GST_QUARK_QUERY_SELECTABLE = 203, GST_QUARK_SELECTABLE = 204, - GST_QUARK_MAX = 205 + GST_QUARK_EVENT_LATENCY_CHANGED = 205, + GST_QUARK_MAX = 206 } GstQuarkId; extern GQuark _priv_gst_quark_table[GST_QUARK_MAX]; From d5070684b1eda0b0c8a729065a841cd17f97aeae Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 4 Dec 2015 11:07:34 +0100 Subject: [PATCH 095/339] gstinfo: Add option to log using system logger [pexhack] Enable feature with --enable-gst-debug-syslog (default is disabled). Use it with setting environment variable GST_DEBUG_SYSLOG. The value of the variable is used as ident for the log message. --- subprojects/gstreamer/gst/gstinfo.c | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/subprojects/gstreamer/gst/gstinfo.c b/subprojects/gstreamer/gst/gstinfo.c index e3872197326..09614ad4878 100644 --- a/subprojects/gstreamer/gst/gstinfo.c +++ b/subprojects/gstreamer/gst/gstinfo.c @@ -99,6 +99,10 @@ #include #include /* G_VA_COPY */ +#ifdef ENABLE_GST_DEBUG_SYSLOG +# include +#endif + #include "gst_private.h" #include "gstutils.h" #include "gstquark.h" @@ -264,6 +268,10 @@ GstDebugCategory *_priv_GST_CAT_PROTECTION = NULL; #ifndef GST_DISABLE_GST_DEBUG +#ifdef ENABLE_GST_DEBUG_SYSLOG +static gboolean _gst_debug_use_syslog = FALSE; +#endif + /* underscore is to prevent conflict with GST_CAT_DEBUG define */ GST_DEBUG_CATEGORY_STATIC (_GST_CAT_DEBUG); @@ -415,6 +423,14 @@ _priv_gst_debug_init (void) gst_debug_add_log_function (gst_debug_log_default, log_file, NULL); } +#ifdef ENABLE_GST_DEBUG_SYSLOG + env = g_getenv ("GST_DEBUG_SYSLOG"); + if (env != NULL && *env != '\0') { + _gst_debug_use_syslog = TRUE; + openlog (env, LOG_NDELAY | LOG_PID, LOG_LOCAL3); + } +#endif + __gst_printf_pointer_extension_set_func (gst_info_printf_pointer_extension_func); @@ -1665,6 +1681,20 @@ gst_debug_log_default (GstDebugCategory * category, GstDebugLevel level, } FFLUSH_DEBUG (log_file); } + +#ifdef ENABLE_GST_DEBUG_SYSLOG + if (_gst_debug_use_syslog) { +#define PRINT_FMT " "PID_FMT" "PTR_FMT" %s "CAT_FMT" %s\n" + syslog (LOG_INFO|LOG_LOCAL3, "%" GST_TIME_FORMAT PRINT_FMT, GST_TIME_ARGS (elapsed), + pid, g_thread_self (), gst_debug_level_get_name (level), + gst_debug_category_get_name (category), file, line, function, obj, + gst_debug_message_get (message)); +#undef PRINT_FMT + } +#endif + + if (object != NULL) + g_free (obj); } /** From 9607167361462ae6b9151f22fab76bdf0430b9f2 Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Tue, 27 May 2014 13:38:08 +0200 Subject: [PATCH 096/339] check: Disable gst_deinit atexit [pexhack] --- subprojects/gstreamer/libs/gst/check/gstcheck.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/subprojects/gstreamer/libs/gst/check/gstcheck.c b/subprojects/gstreamer/libs/gst/check/gstcheck.c index 3ff00be8b15..9e77afb9d06 100644 --- a/subprojects/gstreamer/libs/gst/check/gstcheck.c +++ b/subprojects/gstreamer/libs/gst/check/gstcheck.c @@ -328,7 +328,9 @@ print_plugins (void) static void gst_check_deinit (void) { +#if 0 gst_deinit (); +#endif gst_check_clear_log_filter (); } From 765fc058f411cb15f6dd45d71647af50d3780c77 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 11 Feb 2013 16:31:51 +0100 Subject: [PATCH 097/339] filesrc: add simple loop functionality [pexhack] --- .../gstreamer/plugins/elements/gstfilesrc.c | 23 ++++++++++++++++++- .../gstreamer/plugins/elements/gstfilesrc.h | 3 +++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/subprojects/gstreamer/plugins/elements/gstfilesrc.c b/subprojects/gstreamer/plugins/elements/gstfilesrc.c index 5dfe014169b..bfd3b63f253 100644 --- a/subprojects/gstreamer/plugins/elements/gstfilesrc.c +++ b/subprojects/gstreamer/plugins/elements/gstfilesrc.c @@ -110,11 +110,13 @@ enum }; #define DEFAULT_BLOCKSIZE 4*1024 +#define DEFAULT_LOOP 0 enum { PROP_0, - PROP_LOCATION + PROP_LOCATION, + PROP_LOOP }; static void gst_file_src_finalize (GObject * object); @@ -163,6 +165,11 @@ gst_file_src_class_init (GstFileSrcClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY)); + g_object_class_install_property (gobject_class, PROP_LOOP, + g_param_spec_int ("loop", "Loop", + "How many times to loop when reaching EOF", 0, G_MAXINT, DEFAULT_LOOP, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + gobject_class->finalize = gst_file_src_finalize; gst_element_class_set_static_metadata (gstelement_class, @@ -191,6 +198,9 @@ gst_file_src_init (GstFileSrc * src) src->fd = 0; src->uri = NULL; + src->loop = 0; + src->filesize = G_MAXUINT64; + src->is_regular = FALSE; gst_base_src_set_blocksize (GST_BASE_SRC (src), DEFAULT_BLOCKSIZE); @@ -270,6 +280,9 @@ gst_file_src_set_property (GObject * object, guint prop_id, case PROP_LOCATION: gst_file_src_set_location (src, g_value_get_string (value), NULL); break; + case PROP_LOOP: + src->loop = g_value_get_int (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -290,6 +303,9 @@ gst_file_src_get_property (GObject * object, guint prop_id, GValue * value, case PROP_LOCATION: g_value_set_string (value, src->filename); break; + case PROP_LOOP: + g_value_set_int (value, src->loop); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -318,6 +334,8 @@ gst_file_src_fill (GstBaseSrc * basesrc, guint64 offset, guint length, src = GST_FILE_SRC_CAST (basesrc); + offset = offset % src->filesize; + if (G_UNLIKELY (offset != -1 && src->read_position != offset)) { off_t res; @@ -440,6 +458,9 @@ gst_file_src_get_size (GstBaseSrc * basesrc, guint64 * size) } #endif + src->filesize = *size; + *size = src->filesize * (src->loop + 1); + return TRUE; /* ERROR */ diff --git a/subprojects/gstreamer/plugins/elements/gstfilesrc.h b/subprojects/gstreamer/plugins/elements/gstfilesrc.h index e73cfc3d5ea..b831eac2adc 100644 --- a/subprojects/gstreamer/plugins/elements/gstfilesrc.h +++ b/subprojects/gstreamer/plugins/elements/gstfilesrc.h @@ -63,6 +63,9 @@ struct _GstFileSrc { gboolean seekable; /* whether the file is seekable */ gboolean is_regular; /* whether it's a (symlink to a) regular file */ + + guint64 filesize; /* file size */ + gint loop; /* number of loops before EOS */ }; struct _GstFileSrcClass { From 63ad4a93505d9ec04f604dabb461248f8d590822 Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Tue, 10 Jan 2017 11:42:29 +0100 Subject: [PATCH 098/339] Revert "info: re-eval GST_DEBUG env var for late categories" This reverts commit ee52459a5ed7e78f3796a4737374904194b5a5b1. --- subprojects/gstreamer/gst/gstinfo.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/subprojects/gstreamer/gst/gstinfo.c b/subprojects/gstreamer/gst/gstinfo.c index 09614ad4878..75789a37559 100644 --- a/subprojects/gstreamer/gst/gstinfo.c +++ b/subprojects/gstreamer/gst/gstinfo.c @@ -2447,15 +2447,8 @@ gst_debug_set_threshold_from_string (const gchar * list, gboolean reset) const gchar *category; if (parse_debug_category (values[0], &category) - && parse_debug_level (values[1], &level)) { + && parse_debug_level (values[1], &level)) gst_debug_set_threshold_for_name (category, level); - - /* bump min-level anyway to allow the category to be registered in the - * future still */ - if (level > _gst_debug_min) { - _gst_debug_min = level; - } - } } g_strfreev (values); From bf14f832b8f70ecf72947655c80425398361fc7b Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 31 Mar 2017 16:46:15 +0200 Subject: [PATCH 099/339] gstbasetransform: fix for element going away while receiving a query Before this could lead to the trans->sinkpad and ->srcpad being invalid when used. --- .../libs/gst/base/gstbasetransform.c | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/base/gstbasetransform.c b/subprojects/gstreamer/libs/gst/base/gstbasetransform.c index 0434e98d066..f4998cc1f32 100644 --- a/subprojects/gstreamer/libs/gst/base/gstbasetransform.c +++ b/subprojects/gstreamer/libs/gst/base/gstbasetransform.c @@ -1502,13 +1502,21 @@ gst_base_transform_default_query (GstBaseTransform * trans, GstPad *pad, *otherpad; GstBaseTransformClass *klass; GstBaseTransformPrivate *priv = trans->priv; + GstElement *el = GST_ELEMENT_CAST (trans); + GST_OBJECT_LOCK (el); if (direction == GST_PAD_SRC) { - pad = trans->srcpad; - otherpad = trans->sinkpad; + pad = el->srcpads ? gst_object_ref (el->srcpads->data) : NULL; + otherpad = el->sinkpads ? gst_object_ref (el->sinkpads->data) : NULL; } else { - pad = trans->sinkpad; - otherpad = trans->srcpad; + pad = el->sinkpads ? gst_object_ref (el->sinkpads->data) : NULL; + otherpad = el->srcpads ? gst_object_ref (el->srcpads->data) : NULL; + } + GST_OBJECT_UNLOCK (el); + + if (!pad || !otherpad) { + GST_WARNING_OBJECT (trans, "Our pads are missing!"); + goto done; } klass = GST_BASE_TRANSFORM_GET_CLASS (trans); @@ -1621,6 +1629,10 @@ gst_base_transform_default_query (GstBaseTransform * trans, } done: + if (pad) + gst_object_unref (pad); + if (otherpad) + gst_object_unref (otherpad); return ret; } From 518e4376c40e9897c15e256782bf748edaed0e24 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 6 Jun 2017 14:16:26 +0200 Subject: [PATCH 100/339] libcheck: jump out on failure for CK_FORK=no Or else the next test will just hang waiting to clean up threads from the test that failed. --- .../gstreamer/libs/gst/check/libcheck/check_run.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c b/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c index 2b8a87f5423..82bd83ea88d 100644 --- a/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c +++ b/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c @@ -157,6 +157,14 @@ srunner_run_end (SRunner * sr, enum print_output CK_ATTRIBUTE_UNUSED print_mode) set_fork_status (CK_FORK); } +static int +check_nofork_failure (SRunner * sr) +{ + return (srunner_fork_status (sr) == CK_NOFORK && + (sr->stats->n_errors + sr->stats->n_failed) > 0); +} + + static void srunner_iterate_suites (SRunner * sr, const char *sname, const char *tcname, @@ -205,6 +213,8 @@ srunner_iterate_suites (SRunner * sr, } srunner_run_tcase (sr, tc); + if (check_nofork_failure (sr)) + break; } log_suite_end (sr, s); @@ -252,8 +262,12 @@ srunner_iterate_tcase_tfuns (SRunner * sr, TCase * tc) if (NULL != tr) { srunner_add_failure (sr, tr); log_test_end (sr, tr); + if (check_nofork_failure (sr)) + break; } } + if (check_nofork_failure (sr)) + break; } } From bd2d4fbfcd8df4975f00212debae06832252fae9 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Wed, 14 Jun 2017 09:32:56 +0200 Subject: [PATCH 101/339] pad: Warn when trying to link not compatible caps [pexhack] --- subprojects/gstreamer/gst/gstpad.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/subprojects/gstreamer/gst/gstpad.c b/subprojects/gstreamer/gst/gstpad.c index a83f25ec861..2dc9c0376c4 100644 --- a/subprojects/gstreamer/gst/gstpad.c +++ b/subprojects/gstreamer/gst/gstpad.c @@ -2292,20 +2292,23 @@ gst_pad_link_check_compatible_unlocked (GstPad * src, GstPad * sink, /* if we have caps on both pads we can check the intersection. If one * of the caps is %NULL, we return %TRUE. */ if (G_UNLIKELY (srccaps == NULL || sinkcaps == NULL)) { - if (srccaps) - gst_caps_unref (srccaps); - if (sinkcaps) - gst_caps_unref (sinkcaps); goto done; } compatible = gst_caps_can_intersect (srccaps, sinkcaps); - gst_caps_unref (srccaps); - gst_caps_unref (sinkcaps); done: GST_CAT_DEBUG (GST_CAT_CAPS, "caps are %scompatible", (compatible ? "" : "not ")); + if (!compatible) { + GST_CAT_WARNING (GST_CAT_CAPS, "caps are not compatible: %" GST_PTR_FORMAT + " and %" GST_PTR_FORMAT, srccaps, sinkcaps); + } + + if (srccaps) + gst_caps_unref (srccaps); + if (sinkcaps) + gst_caps_unref (sinkcaps); return compatible; } From 41f65f51a505ce5a64e86a5da48cc8508d3a5195 Mon Sep 17 00:00:00 2001 From: Erlend Graff Date: Wed, 2 Nov 2016 15:45:13 +0100 Subject: [PATCH 102/339] Add priority queue implementation + tests --- subprojects/gstreamer/gst/gstmacros.h | 3 + subprojects/gstreamer/libs/gst/base/base.h | 1 + .../gstreamer/libs/gst/base/gstpriqueue.c | 900 ++++++++++++++++++ .../gstreamer/libs/gst/base/gstpriqueue.h | 124 +++ .../gstreamer/libs/gst/base/meson.build | 3 + tests/check/libs/gstpriqueue.c | 420 ++++++++ 6 files changed, 1451 insertions(+) create mode 100644 subprojects/gstreamer/libs/gst/base/gstpriqueue.c create mode 100644 subprojects/gstreamer/libs/gst/base/gstpriqueue.h create mode 100644 tests/check/libs/gstpriqueue.c diff --git a/subprojects/gstreamer/gst/gstmacros.h b/subprojects/gstreamer/gst/gstmacros.h index b3313e4e370..3216a94f78d 100644 --- a/subprojects/gstreamer/gst/gstmacros.h +++ b/subprojects/gstreamer/gst/gstmacros.h @@ -52,6 +52,9 @@ G_BEGIN_DECLS # endif #endif +#define GST_CONTAINER_OF(addr, type, member) \ + ((type * const) ((guint8 *) (addr) - G_STRUCT_OFFSET (type, member))) + G_END_DECLS #endif /* __GST_MACROS_H__ */ diff --git a/subprojects/gstreamer/libs/gst/base/base.h b/subprojects/gstreamer/libs/gst/base/base.h index 5ad42c11b27..a802a21feca 100644 --- a/subprojects/gstreamer/libs/gst/base/base.h +++ b/subprojects/gstreamer/libs/gst/base/base.h @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include diff --git a/subprojects/gstreamer/libs/gst/base/gstpriqueue.c b/subprojects/gstreamer/libs/gst/base/gstpriqueue.c new file mode 100644 index 00000000000..adc12e02216 --- /dev/null +++ b/subprojects/gstreamer/libs/gst/base/gstpriqueue.c @@ -0,0 +1,900 @@ +/* GStreamer + * Copyright (C) 2016 Pexip AS + * Erlend Graff + * + * gstpriqueue.c: binomial heap implementation of a priority queue + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstpriqueue.h" + +#include +#include +#include +#include + +/** + * SECTION:gstpriqueue + * @title: Priority Queues + * @short_description: collections optimized for dequeuing elements in priority + * order + * + * #GstPriQueue provides the API of a priority queue, where the only supported + * lookup operation is to find/dequeue the element with the highest priority, + * which is defined as the smallest value according to the + * supplied comparison function. + * + * The #GstPriQueue is implemented internally as a binomial heap. This makes it + * possible to find or remove the smallest element with an upper time bound of + * `O(log n)`. Insertions have a worst-case time of `O(log n)`, but an amortized + * asymptotic bound of `O(1)` over `n` consecutive insertions. Deletions are + * supported in `O(log n)`, but deleting an arbitrary element (other than the + * smallest element) is much more expensive than an insertion (up to a constant + * factor of approximately 5 in the worst case). An update operation + * (reinsertion of an element after its value has changed) is also implemented + * in `O(log n)`, and is faster than a corresponding delete + (re-)insert. + * Finally, the binomial heap structure also supports merging two priority + * queues, in `O(log n)` time. + * + * The #GstPriQueue does not handle memory allocation of its elements. Instead, + * the responsibility for allocating and freeing a #GstPriQueueElem, whenever + * it is inserted into or removed from the #GstPriQueue, is given to the caller. + * Typically, the caller will embed the #GstPriQueueElem structure inside a + * containing "parent" structure that will represent the value of the element. + * For example: + * |[ + * typedef struct + * { + * ... + * GstPriQueueElem pq_elem; + * ... + * } SomeValue; + * + * ... + * + * SomeValue *val = create_new_some_value (); + * + * gst_pri_queue_insert (pq, &val->pq_elem); + * ]| + * The GST_CONTAINER_OF() macro defined in gst/gstmacros.h + * can then be used to retrieve the containing `SomeValue` of a + * #GstPriQueueElem: + * |[ + * GstPriQueueElem *elem; + * SomeValue *val; + * + * elem = gst_pri_queue_pop_min (pq); + * if (elem) { + * val = GST_CONTAINER_OF (elem, SomeValue, pq_elem); + * ... + * } + * ]| + * The comparison function will also typically make use of GST_CONTAINER_OF(): + * |[ + * static gint + * compare_some_value_pq_elems (const GstPriQueueElem * a, + * const GstPriQueueElem * b, gpointer user_data) + * { + * SomeValue *val_a = GST_CONTAINER_OF (a, SomeValue, pq_elem); + * SomeValue *val_b = GST_CONTAINER_OF (b, SomeValue, pq_elem); + * + * return compare_some_value (val_a, val_b, user_data); + * } + * + * ... + * + * GstPriQueue *pq; + * + * pq = gst_pri_queue_create (compare_some_value_pq_elems, some_user_data); + * ]| + */ + +#define NODE2ELEM(nodeptr) (GST_CONTAINER_OF ((nodeptr), GstPriQueueElem, node)) +#define ELEM2NODE(elemptr) (&(elemptr)->node) + +typedef struct _GstPriQueueNode GstPriQueueNode; + +/** + * GstPriQueue: (skip) + * + * The #GstPriQueue struct is an opaque data type representing a priority + * queue. + */ +struct _GstPriQueue +{ + GstPriQueueNode *head; + GstPriQueueCompareFunc cmp_func; + gpointer user_data; + gsize size; +}; + +/** + * GstPriQueueElem: (skip) + * + * The #GstPriQueueElem struct represents an element in a #GstPriQueue. + * This struct must be treated as an opaque data type, but has a public + * definition to allow it to be embedded in another "parent" structure + * representing the value of an element. + */ + +/** + * GstPriQueueIter: (skip) + * + * The #GstPriQueueIter struct represents an iterator that can be used to + * iterate over all the elements in a #GstPriQueue. This struct must be treated + * as an opaque data type, but has a public definition to allow it to be + * allocated on the stack. + * + * A #GstPriQueueIter is initialized using gst_pri_queue_iter_init(), and + * gst_pri_queue_iter_next() is used to do the actual iteration. Note that + * iteration over elements in a #GstPriQueue is performed in an arbitrary order, + * and not in priority order. The first element of an + * iteration sequence is not even guaranteed to be the smallest element returned + * by gst_pri_queue_get_min() or gst_pri_queue_pop_min(), or any of the smallest + * elements in the #GstPriQueue if there are multiple such. + */ + +typedef struct +{ + GstPriQueueNode *parent; + GstPriQueueNode **list_pos; + GstPriQueueNode *children_head; + gint order; +} BinomTreePos; + +static inline gint +compare_nodes (const GstPriQueue * pq, GstPriQueueNode * a, GstPriQueueNode * b) +{ + return pq->cmp_func (NODE2ELEM (a), NODE2ELEM (b), pq->user_data); +} + +/* Remove @delnode from the list pointed to by @head. This may change what @head + * points to. + * + * Note: @head MUST be the actual head of the list! + */ +static inline GstPriQueueNode ** +list_remove_node (GstPriQueueNode ** head, GstPriQueueNode * delnode) +{ + GstPriQueueNode **pnext; + GstPriQueueNode *node; + + for (pnext = head; (node = *pnext) != delnode;) + pnext = &node->next; + + *pnext = node->next; + return pnext; +} + +/* Insert node at given position in list */ +static inline void +list_insert_node (GstPriQueueNode ** pnext, GstPriQueueNode * insnode) +{ + insnode->next = *pnext; + *pnext = insnode; +} + +static inline void +init_node (GstPriQueueNode * node) +{ + node->parent = NULL; + node->children_head = NULL; + node->next = NULL; + node->order = 0; +} + +/* Merge two binomial trees of the same order `n` into a binomial tree of order + * `(n + 1)`. + */ +static inline GstPriQueueNode * +merge_tree (const GstPriQueue * pq, GstPriQueueNode * a, GstPriQueueNode * b) +{ + GstPriQueueNode *new_root, *new_subtree; + + if (compare_nodes (pq, a, b) <= 0) { + new_root = a; + new_subtree = b; + } else { + new_root = b; + new_subtree = a; + } + + list_insert_node (&new_root->children_head, new_subtree); + new_subtree->parent = new_root; + new_root->order++; + + return new_root; +} + +/* Reverses list, and sets parent to %NULL. */ +static inline GstPriQueueNode * +subtree_list_to_heap_list (GstPriQueueNode * head) +{ + GstPriQueueNode *node, *new_head, *next; + + new_head = NULL; + node = head; + while (node) { + next = node->next; + node->next = new_head; + new_head = node; + node->parent = NULL; + node = next; + } + + return new_head; +} + +static inline GstPriQueueNode ** +get_containing_list (GstPriQueue * pq, GstPriQueueNode * node) +{ + return node->parent ? &node->parent->children_head : &pq->head; +} + +static inline GstPriQueueNode ** +remove_node_from_containing_list (GstPriQueue * pq, GstPriQueueNode * delnode) +{ + return list_remove_node (get_containing_list (pq, delnode), delnode); +} + +/* Add a single node to the given binomial heap list (list of root binomial + * trees). This will likely change the head of the binomial heap list. + */ +static inline void +binom_heap_list_add_node (GstPriQueue * pq, GstPriQueueNode ** head, + GstPriQueueNode * insnode) +{ + GstPriQueueNode *next; + + while ((next = *head) && insnode->order == next->order) { + (void) list_remove_node (head, next); + insnode = merge_tree (pq, next, insnode); + } + + list_insert_node (head, insnode); +} + +/* Merge the binomial heap list (list of root binomial trees) pointed to by + * @head_b (B-list) into @pq's binomial heap list (A-list). This will modify + * the A-list, and destroy the B-list. + */ +static void +binom_heap_union (GstPriQueue * pq, GstPriQueueNode * head_b) +{ + GstPriQueueNode **pnext_a; + GstPriQueueNode *next_a, *node; + + pnext_a = &pq->head; + while ((next_a = *pnext_a) && head_b) { + if (head_b->order > next_a->order) { + pnext_a = &next_a->next; + } else if (head_b->order < next_a->order) { + /* Remove head of B-list, and insert into A-list */ + node = head_b; + (void) list_remove_node (&head_b, node); + list_insert_node (pnext_a, node); + } else { + /* Remove node from A-list, and add to B-list */ + node = next_a; + (void) list_remove_node (pnext_a, node); + binom_heap_list_add_node (pq, &head_b, node); + } + } + + if (head_b) { + /* A-list exhausted, append rest of B-list to end */ + *pnext_a = head_b; + } +} + +static inline void +remove_tree_node (GstPriQueue * pq, GstPriQueueNode * delnode, + BinomTreePos * pos) +{ + pos->parent = delnode->parent; + pos->list_pos = remove_node_from_containing_list (pq, delnode); + pos->children_head = delnode->children_head; + pos->order = delnode->order; +} + +/* Note: this function does not set insnode->parent. */ +static inline void +insert_tree_node (BinomTreePos pos, GstPriQueueNode * insnode) +{ + GstPriQueueNode *child; + + for (child = pos.children_head; child; child = child->next) + child->parent = insnode; + + insnode->order = pos.order; + list_insert_node (pos.list_pos, insnode); + insnode->children_head = pos.children_head; +} + +static inline gboolean +should_decrease (GstPriQueue * pq, GstPriQueueNode * parent, + GstPriQueueNode * node, gboolean node_is_minus_inf) +{ + return parent && (node_is_minus_inf || compare_nodes (pq, node, parent) < 0); +} + +/* Try to move the given node upwards in its binomial tree until the min-heap + * invariant is satisfied. If @is_minus_if is %TRUE, the node will end up at + * the root of its binomial tree, otherwise its new position is determined by + * its value according to the comparison function. + * + * Returns %TRUE if @node was moved upwards in its binomial tree, else + * %FALSE. + */ +static inline gboolean +decrease_key (GstPriQueue * pq, GstPriQueueNode * node, gboolean is_minus_inf) +{ + BinomTreePos current_pos, parent_pos; + + if (!should_decrease (pq, node->parent, node, is_minus_inf)) + return FALSE; + + remove_tree_node (pq, node, ¤t_pos); + + do { + remove_tree_node (pq, current_pos.parent, &parent_pos); + insert_tree_node (current_pos, current_pos.parent); + + /* Check if current_pos.parent was inserted at head of its new containing + * list. If so, we must manually update parent_pos.children_head. + */ + if (parent_pos.children_head == current_pos.parent->next) + parent_pos.children_head = current_pos.parent; + + current_pos = parent_pos; + } while (should_decrease (pq, current_pos.parent, node, is_minus_inf)); + + insert_tree_node (current_pos, node); + node->parent = current_pos.parent; + + return TRUE; +} + +/* Update node's position in its own binomial subtree after its value might + * have changed. + */ +static inline void +increase_key (GstPriQueue * pq, GstPriQueueNode * node) +{ + GstPriQueueNode **list_pos; + GstPriQueueNode *parent, *head; + + parent = node->parent; + list_pos = remove_node_from_containing_list (pq, node); + + head = subtree_list_to_heap_list (node->children_head); + node->children_head = NULL; + node->order = 0; + binom_heap_list_add_node (pq, &head, node); + + list_insert_node (list_pos, head); + head->parent = parent; +} + +static inline void +remove_heap_root (GstPriQueue * pq, GstPriQueueNode * delnode) +{ + (void) list_remove_node (&pq->head, delnode); + binom_heap_union (pq, subtree_list_to_heap_list (delnode->children_head)); +} + +static inline GstPriQueueNode * +get_min_root (const GstPriQueue * pq) +{ + GstPriQueueNode *node, *min_node; + + node = pq->head; + if (!node) + return NULL; + + min_node = node; + for (node = node->next; node; node = node->next) { + if (compare_nodes (pq, node, min_node) < 0) + min_node = node; + } + + return min_node; +} + +/* + * Debug helper functions + */ + +static void +write_dot_node (const GstPriQueueNode * node, FILE * out, + GstPriQueueWriteElem write_elem_func, gpointer user_data) +{ + fprintf (out, " %" G_GUINTPTR_FORMAT " [label=\"", (guintptr) node); + write_elem_func (out, NODE2ELEM (node), user_data); + fprintf (out, "\"];\n"); +} + +static void +_write_dot_children (const GstPriQueue * pq, GstPriQueueNode * root, FILE * out, + GstPriQueueWriteElem write_elem_func, gpointer user_data) +{ + GstPriQueueNode *node; + + for (node = root->children_head; node; node = node->next) { + write_dot_node (node, out, write_elem_func, user_data); + + fprintf (out, " %" G_GUINTPTR_FORMAT, (guintptr) root); + fprintf (out, " -> %" G_GUINTPTR_FORMAT, (guintptr) node); + fprintf (out, " [color=red];\n"); + + if (node->parent) { + fprintf (out, " %" G_GUINTPTR_FORMAT, (guintptr) node); + fprintf (out, " -> %" G_GUINTPTR_FORMAT, (guintptr) node->parent); + fprintf (out, " [color=blue];\n"); + } + + if (node->children_head) + _write_dot_children (pq, node, out, write_elem_func, user_data); + } +} + +static void +_write_dot_tree (const GstPriQueue * pq, GstPriQueueNode * tree, FILE * out, + GstPriQueueWriteElem write_elem_func, gpointer user_data) +{ + /* Binomial heap list (list of root binomial trees) is in increasing order, + * but we want the trees to be in decreasing order (from left to right) in the + * DOT file, so recurse to get a kind of post-order traversal. + */ + if (tree->next) + _write_dot_tree (pq, tree->next, out, write_elem_func, user_data); + + write_dot_node (tree, out, write_elem_func, user_data); + _write_dot_children (pq, tree, out, write_elem_func, user_data); +} + +static gboolean +_binom_tree_is_invariant (const GstPriQueue * pq, GstPriQueueNode * root, + gsize * size) +{ + GstPriQueueNode *child; + gsize subtree_size; + gint res, num_children, expected_order; + + *size = 1; + num_children = 0; + expected_order = root->order; + + for (child = root->children_head; child; child = child->next) { + num_children++; + expected_order--; + + if (G_UNLIKELY (child->order != expected_order)) + return FALSE; + + res = compare_nodes (pq, root, child); + if (G_UNLIKELY (res > 0)) + return FALSE; + + if (G_UNLIKELY (child->parent != root)) + return FALSE; + + if (G_UNLIKELY (!_binom_tree_is_invariant (pq, child, &subtree_size))) + return FALSE; + + *size += subtree_size; + } + + if (G_UNLIKELY (expected_order != 0)) + return FALSE; + + if (G_UNLIKELY (num_children != root->order)) + return FALSE; + + return TRUE; +} + +static inline gboolean +is_heap_list_order_increasing (const GstPriQueueNode * head) +{ + const GstPriQueueNode *node; + + for (node = head; node && node->next; node = node->next) { + if (node->order >= node->next->order) + return FALSE; + } + + return TRUE; +} + +/* + * Public API + */ + +/** + * gst_pri_queue_create: (skip) + * @cmp_func: the #GstPriQueueCompareFunc used to compare elements in the + * #GstPriQueue. It should return a number < 0 if the first element is smaller + * (i.e. has a higher priority) than the second. + * @user_data: user data passed to @cmp_func + * + * Creates a new #GstPriQueue. The @cmp_func is used to determine which of any + * two elements in the #GstPriQueue is smaller (i.e. has the highest priority). + * + * Returns: a new #GstPriQueue + */ + /** + * GstPriQueueCompareFunc: (skip) + * @elem_a: a #GstPriQueueElem + * @elem_b: a #GstPriQueueElem to compare with + * @user_data: user data + * + * Specifies the type of a comparison function used to compare two elements in + * the #GstPriQueue. The function should return a negative integer if the first + * element is smaller (i.e. has a higher priority) than the second, 0 if they + * are equal, or a positive integer if the first element is larger (i.e. has a + * lower priority). + * + * Returns: negative value if @elem_a < @elem_b ; zero if @elem_a = @elem_b ; + * positive value if @elem_a > @elem_b + */ +GstPriQueue * +gst_pri_queue_create (GstPriQueueCompareFunc cmp_func, gpointer user_data) +{ + GstPriQueue *pq; + + pq = g_new (GstPriQueue, 1); + pq->cmp_func = cmp_func; + pq->user_data = user_data; + pq->head = NULL; + pq->size = 0; + + return pq; +} + +/** + * gst_pri_queue_destroy: (skip) + * @pq: the #GstPriQueue to destroy + * @elem_destroy_func: (nullable): a function that is called for each + * element in the #GstPriQueue before it is destroyed (e.g. to free the memory + * allocated for the element). + * + * Destroys the #GstPriQueue. + */ +void +gst_pri_queue_destroy (GstPriQueue * pq, GDestroyNotify elem_destroy_func) +{ + GstPriQueueElem *elem; + GstPriQueueIter iter; + + gst_pri_queue_iter_init (&iter, pq); + while (gst_pri_queue_iter_next (&iter, &elem)) { + if (elem_destroy_func) + elem_destroy_func (elem); + else + init_node (ELEM2NODE (elem)); + } + + g_free (pq); +} + +/** + * gst_pri_queue_size: (skip) + * @pq: a #GstPriQueue + * + * Returns the number of elements in the #GstPriQueue. This function is + * implemented in `O(1)` time. + * + * Returns: the number of elements in the #GstPriQueue + */ +gsize +gst_pri_queue_size (const GstPriQueue * pq) +{ + return pq->size; +} + +/** + * gst_pri_queue_insert: (skip) + * @pq: a #GstPriQueue + * @elem: the #GstPriQueueElem to be inserted + * + * Inserts an element into the #GstPriQueue. This function should only be called + * for elements that are not already inserted into the #GstPriQueue. + */ +void +gst_pri_queue_insert (GstPriQueue * pq, GstPriQueueElem * elem) +{ + GstPriQueueNode *node; + + node = ELEM2NODE (elem); + init_node (node); + binom_heap_list_add_node (pq, &pq->head, node); + pq->size++; +} + +/** + * gst_pri_queue_remove: (skip) + * @pq: a #GstPriQueue + * @elem: the #GstPriQueueElem to remove + * + * Removes a #GstPriQueueElem from the #GstPriQueue. This function should only + * be called for elements that are inserted into the #GstPriQueue. + */ +void +gst_pri_queue_remove (GstPriQueue * pq, GstPriQueueElem * elem) +{ + GstPriQueueNode *delnode; + + delnode = ELEM2NODE (elem); + (void) decrease_key (pq, delnode, TRUE); + remove_heap_root (pq, delnode); + pq->size--; + + init_node (delnode); +} + +/** + * gst_pri_queue_update: (skip) + * @pq: a #GstPriQueue + * @elem: a #GstPriQueueElem that has changed its associated value (priority) + * + * Updates an element's position in the #GstPriQueue after its value (priority) + * has changed such that the priority queue's comparison function may return a + * different value when comparing it to other elements. This function should + * only be called for elements that are inserted into the #GstPriQueue. + * + * Changing the value associated with a #GstPriQueueElem without calling + * gst_pri_queue_update() afterwards will most likely lead to the #GstPriQueue + * becoming invalid. + */ +void +gst_pri_queue_update (GstPriQueue * pq, GstPriQueueElem * elem) +{ + GstPriQueueNode *node; + + node = ELEM2NODE (elem); + if (!decrease_key (pq, node, FALSE)) + increase_key (pq, node); +} + +/** + * gst_pri_queue_get_min: (skip) + * @pq: a #GstPriQueue + * + * Returns the #GstPriQueueElem with the smallest value (i.e highest priority) + * without dequeuing it from the #GstPriQueue. If the #GstPriQueue contains + * multiple elements with the same (smallest) value, the returned + * #GstPriQueueElem will be one of these. Two consecutive calls to + * gst_pri_queue_get_min() are guaranteed to return the same #GstPriQueueElem + * if the #GstPriQueue has not been modified in between the calls. + * + * Returns: the smallest #GstPriQueueElem in the #GstPriQueue, or %NULL if the + * #GstPriQueue is empty. + */ +GstPriQueueElem * +gst_pri_queue_get_min (const GstPriQueue * pq) +{ + GstPriQueueNode *node; + + node = get_min_root (pq); + return NODE2ELEM (node); +} + +/** + * gst_pri_queue_pop_min: (skip) + * @pq: a #GstPriQueue + * + * Dequeues the #GstPriQueueElem with the smallest value (i.e highest priority) + * from the #GstPriQueue. If the #GstPriQueue contains multiple elements with + * the same (smallest) value, the dequeued #GstPriQueueElem will be one of + * these. + * + * Returns: the smallest #GstPriQueueElem in the #GstPriQueue, or %NULL if the + * #GstPriQueue is empty. + */ +GstPriQueueElem * +gst_pri_queue_pop_min (GstPriQueue * pq) +{ + GstPriQueueNode *delnode; + + delnode = get_min_root (pq); + if (!delnode) + return NULL; + + remove_heap_root (pq, delnode); + pq->size--; + + init_node (delnode); + return NODE2ELEM (delnode); +} + +/** + * gst_pri_queue_meld: (skip) + * @pqa: a #GstPriQueue + * @pqb: a #GstPriQueue to merge with + * + * Merge two priority queues. The second #GstPriQueue @pqb will be destroyed, + * and all its elements will be inserted into the first #GstPriQueue @pqa. This + * function is implemented in `O(log n)` time. + * + * Returns: @pqa after the elements from @pqb have been merged into it. + */ +GstPriQueue * +gst_pri_queue_meld (GstPriQueue * pqa, GstPriQueue * pqb) +{ + binom_heap_union (pqa, pqb->head); + pqa->size += pqb->size; + + pqb->head = NULL; + gst_pri_queue_destroy (pqb, NULL); + + return pqa; +} + +/* + * Iterator API + */ + +/** + * gst_pri_queue_iter_init: (skip) + * @iter: a #GstPriQueueIter + * @pq: a #GstPriQueue + * + * Initialize an iterator over the elements in @pq. + * + * Note that the iterator becomes invalid if the #GstPriQueue @pq is modified. + */ +void +gst_pri_queue_iter_init (GstPriQueueIter * iter, GstPriQueue * pq) +{ + iter->node = pq->head; +} + +/** + * gst_pri_queue_iter_next: (skip) + * @iter: a #GstPriQueueIter + * @elem: (out) (nullable) (optional): a location to store the #GstPriQueueElem, + * or %NULL + * + * Retrieves the #GstPriQueueElem at the iterator's current position, and + * advances the iterator. + * + * Example: + * |[ + * GstPriQueueIter iter; + * GstPriQueueElem *elem; + * + * gst_pri_queue_iter_init (&iter, pq); + * while (gst_pri_queue_iter_next (&iter, &elem)) { + * SomeValue *value = GST_CONTAINER_OF (elem, SomeValue, pq_elem); + * ... + * } + * ]| + * + * Note that iteration over elements in a #GstPriQueue is performed in an + * arbitrary order, and not in priority order. The first + * element of an iteration sequence is not even guaranteed to be the smallest + * element returned by gst_pri_queue_get_min() or gst_pri_queue_pop_min(), or + * any of the smallest elements in the #GstPriQueue if there are multiple such. + * + * Returns: %TRUE if a #GstPriQueueElem could be retrieved, or %FALSE if the end + * of the #GstPriQueue has been reached. + */ +gboolean +gst_pri_queue_iter_next (GstPriQueueIter * iter, GstPriQueueElem ** elem) +{ + GstPriQueueNode *node, *next; + + node = iter->node; + if (!node) + return FALSE; + + if (node->children_head) { + next = node->children_head; + } else { + for (next = node; next && !next->next;) + next = next->parent; + + if (next) + next = next->next; + } + + iter->node = next; + if (elem) + *elem = NODE2ELEM (node); + + return TRUE; +} + +/* + * Debug API + */ + +/** + * gst_pri_queue_is_valid: (skip) + * @pq: a #GstPriQueue + * + * Check that the invariants of the internal data structures have not been + * violated. + * + * This function is intended for testing and/or debugging purposes only. + * + * + * Returns: %TRUE if no invariants have been violated, else %FALSE. + */ +gboolean +gst_pri_queue_is_valid (const GstPriQueue * pq) +{ + GstPriQueueNode *heap; + gsize size, heap_size; + + if (G_UNLIKELY (!is_heap_list_order_increasing (pq->head))) + return FALSE; + + size = 0; + for (heap = pq->head; heap; heap = heap->next) { + if (G_UNLIKELY (!_binom_tree_is_invariant (pq, heap, &heap_size))) + return FALSE; + + size += heap_size; + + if (G_UNLIKELY (heap->parent != NULL)) + return FALSE; + } + + if (G_UNLIKELY (size != pq->size)) + return FALSE; + + return TRUE; +} + +/** + * gst_pri_queue_write_dot_file: (skip) + * @pq: a #GstPriQueue + * @out: the output stream to which the DOT file is written + * @write_elem_func: a #GstPriQueueWriteElem used as callback to write elements + * to the DOT file + * @user_data: user data passed to @write_elem_func + * + * Write a DOT representation of the internal data structures to the given + * output stream. + * + * This function is intended for testing and/or debugging purposes only. + * + */ + /** + * GstPriQueueWriteElem: (skip) + * @out: the #FILE to where the element is written + * @elem: the element to write + * @user_data: user data + * + * A #GstPriQueueWriteElem is a callback supplied to + * gst_pri_queue_write_dot_file() for writing elements to a DOT file. + * + * Returns: the number of bytes written. + */ +void +gst_pri_queue_write_dot_file (const GstPriQueue * pq, FILE * out, + GstPriQueueWriteElem write_elem_func, gpointer user_data) +{ + fprintf (out, "digraph graphname {\n"); + + if (pq->head) + _write_dot_tree (pq, pq->head, out, write_elem_func, user_data); + + fprintf (out, "}\n"); +} diff --git a/subprojects/gstreamer/libs/gst/base/gstpriqueue.h b/subprojects/gstreamer/libs/gst/base/gstpriqueue.h new file mode 100644 index 00000000000..d5e54766dde --- /dev/null +++ b/subprojects/gstreamer/libs/gst/base/gstpriqueue.h @@ -0,0 +1,124 @@ +/* GStreamer + * Copyright (C) 2016 Pexip AS + * Erlend Graff + * + * gstpriqueue.h: binomial heap implementation of a priority queue + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_PRIQUEUE_H__ +#define __GST_PRIQUEUE_H__ + +#include +#include + +G_BEGIN_DECLS + +typedef struct _GstPriQueue GstPriQueue; +typedef struct _GstPriQueueElem GstPriQueueElem; +typedef struct _GstPriQueueIter GstPriQueueIter; + +typedef gint (* GstPriQueueCompareFunc) (const GstPriQueueElem * elem_a, + const GstPriQueueElem * elem_b, + gpointer user_data); + +struct _GstPriQueueNode +{ + /*< private >*/ + struct _GstPriQueueNode *parent; + struct _GstPriQueueNode *children_head; + struct _GstPriQueueNode *next; + gint order; +}; + +struct _GstPriQueueElem +{ + /*< private >*/ + struct _GstPriQueueNode node; +}; + +struct _GstPriQueueIter +{ + /*< private >*/ + struct _GstPriQueueNode *node; +}; + +GstPriQueue * +gst_pri_queue_create (GstPriQueueCompareFunc cmp_func, + gpointer user_data); + +void +gst_pri_queue_destroy (GstPriQueue * pq, + GDestroyNotify elem_destroy_func); + +gsize +gst_pri_queue_size (const GstPriQueue * pq); + +void +gst_pri_queue_insert (GstPriQueue * pq, + GstPriQueueElem * elem); + +void +gst_pri_queue_remove (GstPriQueue * pq, + GstPriQueueElem * elem); + +void +gst_pri_queue_update (GstPriQueue * pq, + GstPriQueueElem * elem); + +GstPriQueueElem * +gst_pri_queue_get_min (const GstPriQueue * pq); + +GstPriQueueElem * +gst_pri_queue_pop_min (GstPriQueue * pq); + +GstPriQueue * +gst_pri_queue_meld (GstPriQueue * pqa, + GstPriQueue * pqb); + +/* + * Iterator API + */ + +void +gst_pri_queue_iter_init (GstPriQueueIter * iter, + GstPriQueue * pq); + +gboolean +gst_pri_queue_iter_next (GstPriQueueIter * iter, + GstPriQueueElem ** elem); + +/* + * Debug API + */ + +typedef gint (* GstPriQueueWriteElem) (FILE * out, + const GstPriQueueElem * elem, + gpointer user_data); + +gboolean +gst_pri_queue_is_valid (const GstPriQueue * pq); + +void +gst_pri_queue_write_dot_file (const GstPriQueue * pq, + FILE * out, + GstPriQueueWriteElem write_elem_func, + gpointer user_data); + +G_END_DECLS + +#endif /* __GST_PRIQUEUE_H__ */ diff --git a/subprojects/gstreamer/libs/gst/base/meson.build b/subprojects/gstreamer/libs/gst/base/meson.build index e08254d929c..9c85dd86166 100644 --- a/subprojects/gstreamer/libs/gst/base/meson.build +++ b/subprojects/gstreamer/libs/gst/base/meson.build @@ -12,6 +12,7 @@ gst_base_sources = files( 'gstcollectpads.c', 'gstdataqueue.c', 'gstflowcombiner.c', + 'gstpriqueue.c', 'gstpushsrc.c', 'gstqueuearray.c', 'gsttypefindhelper.c', @@ -33,6 +34,7 @@ gst_base_headers = files( 'gstcollectpads.h', 'gstdataqueue.h', 'gstflowcombiner.h', + 'gstpriqueue.h', 'gstpushsrc.h', 'gstqueuearray.h', 'gsttypefindhelper.h', @@ -107,6 +109,7 @@ install_headers('base.h', 'gstcollectpads.h', 'gstdataqueue.h', 'gstflowcombiner.h', + 'gstpriqueue.h', 'gstpushsrc.h', 'gstqueuearray.h', 'gsttypefindhelper.h', diff --git a/tests/check/libs/gstpriqueue.c b/tests/check/libs/gstpriqueue.c new file mode 100644 index 00000000000..30d40c6a7e6 --- /dev/null +++ b/tests/check/libs/gstpriqueue.c @@ -0,0 +1,420 @@ +/* GStreamer + * Copyright (C) 2016 Pexip AS + * Erlend Graff + * + * gstpriqueue.c: unit tests for GstPriQueue + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include +#include +#include + +struct item_data +{ + gint key; + gint inserted; + GstPriQueueElem pq_elem; +}; + +#define ELEM_TO_ITEM(elem) (GST_CONTAINER_OF (elem, struct item_data, pq_elem)) + +static gint +compare_items (const GstPriQueueElem * elem_a, const GstPriQueueElem * elem_b, + G_GNUC_UNUSED gpointer user_data) +{ + return ELEM_TO_ITEM (elem_a)->key - ELEM_TO_ITEM (elem_b)->key; +} + +static gint +compare_items_with_check (const GstPriQueueElem * elem_a, + const GstPriQueueElem * elem_b, gpointer user_data) +{ + fail_unless_equals_int (-12345, GPOINTER_TO_INT (user_data)); + return compare_items (elem_a, elem_b, NULL); +} + +GST_START_TEST (test_random_modifications) +{ + const gint num_items = 1023; + const gint num_iter = 500; + const gint key_min = -1000; + const gint key_max = 1000; + struct item_data *items; + GstPriQueueElem *lookup_elem; + GstPriQueue *pq; + GRand *rand; + gint i, minidx, idx, num_inserted; + + rand = g_rand_new_with_seed (0); + items = g_new0 (struct item_data, num_items); + + pq = gst_pri_queue_create (compare_items_with_check, + GINT_TO_POINTER (-12345)); + + num_inserted = 0; + for (i = 0; i < num_iter; i++) { + /* Ensure valid */ + fail_unless (gst_pri_queue_is_valid (pq)); + + /* Test get_min and size */ + + fail_unless_equals_int (num_inserted, gst_pri_queue_size (pq)); + + minidx = -1; + lookup_elem = gst_pri_queue_get_min (pq); + fail_unless_equals_int (! !num_inserted, ! !lookup_elem); + + if (num_inserted) { + minidx = ELEM_TO_ITEM (lookup_elem) - items; + for (idx = 0; idx < num_items; idx++) { + /* There may be multiple "smallest" elements (i.e. other elements may + * have the same priority/key as the one we just retrieved), hence the + * use of `>=`. + */ + if (items[idx].inserted) + fail_unless (items[idx].key >= items[minidx].key); + } + } + + /* Test insert, remove, and update */ + + idx = g_rand_int_range (rand, 0, num_items); + + if (!items[idx].inserted) { + /* Just to prevent insertions from dominating the types of operations + * performed. + */ + if (g_rand_int_range (rand, 0, 5) != 0) { + i--; + continue; + } + + items[idx].key = g_rand_int_range (rand, key_min, key_max); + gst_pri_queue_insert (pq, &items[idx].pq_elem); + items[idx].inserted = TRUE; + num_inserted++; + } else { + switch (g_rand_int_range (rand, 0, 4)) { + case 0: + gst_pri_queue_remove (pq, &items[idx].pq_elem); + items[idx].inserted = FALSE; + num_inserted--; + break; + case 1: + lookup_elem = gst_pri_queue_pop_min (pq); + fail_unless_equals_pointer (&items[minidx].pq_elem, lookup_elem); + items[minidx].inserted = FALSE; + num_inserted--; + break; + case 2: + gst_pri_queue_remove (pq, &items[idx].pq_elem); + items[idx].key = g_rand_int_range (rand, key_min, key_max); + gst_pri_queue_insert (pq, &items[idx].pq_elem); + break; + case 3: + items[idx].key = g_rand_int_range (rand, key_min, key_max); + gst_pri_queue_update (pq, &items[idx].pq_elem); + break; + default: + g_assert_not_reached (); + break; + } + } + } + + while (TRUE) { + /* Ensure valid */ + fail_unless (gst_pri_queue_is_valid (pq)); + + /* Test get_min and size */ + + fail_unless_equals_int (num_inserted, gst_pri_queue_size (pq)); + + lookup_elem = gst_pri_queue_get_min (pq); + fail_unless_equals_int (! !num_inserted, ! !lookup_elem); + + if (num_inserted) { + minidx = ELEM_TO_ITEM (lookup_elem) - items; + for (idx = 0; idx < num_items; idx++) { + /* There may be multiple "smallest" elements (i.e. other elements may + * have the same priority/key as the one we just retrieved), hence the + * use of `>=`. + */ + if (items[idx].inserted) + fail_unless (items[idx].key >= items[minidx].key); + } + } + + if (!num_inserted) + break; + + /* Test remove and update */ + + /* Find random inserted item */ + idx = g_rand_int_range (rand, 0, num_items); + while (!items[idx].inserted) + idx = (idx + 1) % num_items; + + switch (g_rand_int_range (rand, 0, 2)) { + case 0: + gst_pri_queue_remove (pq, &items[idx].pq_elem); + items[idx].inserted = FALSE; + num_inserted--; + break; + case 1: + items[idx].key = g_rand_int_range (rand, key_min, key_max); + gst_pri_queue_update (pq, &items[idx].pq_elem); + break; + default: + g_assert_not_reached (); + break; + } + } + + gst_pri_queue_destroy (pq, NULL); + + g_free (items); + g_rand_free (rand); +} + +GST_END_TEST; + +#define NUM_ITEMS 127 + +static gint +compare_pointers (const GstPriQueueElem * elem_a, + const GstPriQueueElem * elem_b, G_GNUC_UNUSED gpointer user_data) +{ + return elem_a - elem_b; +} + +GST_START_TEST (test_sorted_insertion) +{ + GstPriQueueElem *elems; + GstPriQueue *pq; + gint i; + + elems = g_new (GstPriQueueElem, NUM_ITEMS); + + pq = gst_pri_queue_create (compare_pointers, NULL); + for (i = 0; i < NUM_ITEMS; i++) { + gst_pri_queue_insert (pq, &elems[i]); + fail_unless_equals_int (i + 1, gst_pri_queue_size (pq)); + fail_unless_equals_pointer (&elems[0], gst_pri_queue_get_min (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + gst_pri_queue_destroy (pq, NULL); + + pq = gst_pri_queue_create (compare_pointers, NULL); + for (i = NUM_ITEMS - 1; i >= 0; i--) { + gst_pri_queue_insert (pq, &elems[i]); + fail_unless_equals_int (NUM_ITEMS - i, gst_pri_queue_size (pq)); + fail_unless_equals_pointer (&elems[i], gst_pri_queue_get_min (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + gst_pri_queue_destroy (pq, NULL); + + g_free (elems); +} + +GST_END_TEST; + +GST_START_TEST (test_sorted_removal) +{ + GstPriQueueElem *elems; + GstPriQueue *pq; + gint i; + + elems = g_new (GstPriQueueElem, NUM_ITEMS); + pq = gst_pri_queue_create (compare_pointers, NULL); + + for (i = 0; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pq, &elems[i]); + + for (i = 0; i < NUM_ITEMS; i++) { + fail_unless_equals_pointer (&elems[i], gst_pri_queue_get_min (pq)); + gst_pri_queue_remove (pq, &elems[i]); + fail_unless_equals_int (NUM_ITEMS - 1 - i, gst_pri_queue_size (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + + fail_unless_equals_pointer (NULL, gst_pri_queue_get_min (pq)); + + for (i = 0; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pq, &elems[i]); + + for (i = NUM_ITEMS - 1; i >= 0; i--) { + fail_unless_equals_pointer (&elems[0], gst_pri_queue_get_min (pq)); + gst_pri_queue_remove (pq, &elems[i]); + fail_unless_equals_int (i, gst_pri_queue_size (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + + fail_unless_equals_pointer (NULL, gst_pri_queue_get_min (pq)); + + gst_pri_queue_destroy (pq, NULL); + g_free (elems); +} + +GST_END_TEST; + +GST_START_TEST (test_pop_min) +{ + GstPriQueueElem *elems; + GstPriQueue *pq; + gint i; + + elems = g_new (GstPriQueueElem, NUM_ITEMS); + pq = gst_pri_queue_create (compare_pointers, NULL); + + for (i = 0; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pq, &elems[i]); + + for (i = 0; i < NUM_ITEMS; i++) { + fail_unless_equals_pointer (&elems[i], gst_pri_queue_pop_min (pq)); + fail_unless_equals_int (NUM_ITEMS - 1 - i, gst_pri_queue_size (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + + fail_unless_equals_pointer (NULL, gst_pri_queue_pop_min (pq)); + + gst_pri_queue_destroy (pq, NULL); + g_free (elems); +} + +GST_END_TEST; + +static void +increase_inserted (gpointer elem) +{ + struct item_data *item = ELEM_TO_ITEM (elem); + item->inserted += 1; +} + +GST_START_TEST (test_destroy) +{ + struct item_data *items; + GstPriQueue *pq; + gint i; + + items = g_new (struct item_data, NUM_ITEMS); + + pq = gst_pri_queue_create (compare_items, NULL); + for (i = 0; i < NUM_ITEMS; i++) { + /* Note: we abuse the `inserted` field to keep track of how many times each + * the destroy func is called for each element. + */ + items[i].inserted = 0; + items[i].key = i; + gst_pri_queue_insert (pq, &items[i].pq_elem); + } + + gst_pri_queue_destroy (pq, increase_inserted); + + for (i = 0; i < NUM_ITEMS; i++) { + /* Each element should be visited by destroy func exactly once */ + fail_unless_equals_int (1, items[i].inserted); + } + + g_free (items); +} + +GST_END_TEST; + +GST_START_TEST (test_meld) +{ + GstPriQueueElem *elems; + GstPriQueue *pqa, *pqb, *pq_meld; + gint i; + + elems = g_new (GstPriQueueElem, NUM_ITEMS); + pqa = gst_pri_queue_create (compare_pointers, NULL); + pqb = gst_pri_queue_create (compare_pointers, NULL); + + for (i = 0; i < NUM_ITEMS / 2; i++) + gst_pri_queue_insert (pqa, &elems[i]); + + for (; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pqb, &elems[i]); + + pq_meld = gst_pri_queue_meld (pqa, pqb); + fail_unless_equals_pointer (pqa, pq_meld); + + fail_unless_equals_int (NUM_ITEMS, gst_pri_queue_size (pqa)); + fail_unless (gst_pri_queue_is_valid (pqa)); + + for (i = 0; i < NUM_ITEMS; i++) { + fail_unless_equals_pointer (&elems[i], gst_pri_queue_pop_min (pqa)); + fail_unless_equals_int (NUM_ITEMS - 1 - i, gst_pri_queue_size (pqa)); + fail_unless (gst_pri_queue_is_valid (pqa)); + } + + gst_pri_queue_destroy (pqa, NULL); + + g_free (elems); +} + +GST_END_TEST; + +GST_START_TEST (test_iter) +{ + struct item_data *items; + GstPriQueueElem *elem; + GstPriQueueIter iter; + GstPriQueue *pq; + gint i; + + items = g_new0 (struct item_data, NUM_ITEMS); + pq = gst_pri_queue_create (compare_items, NULL); + + for (i = 0; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pq, &items[i].pq_elem); + + gst_pri_queue_iter_init (&iter, pq); + while (gst_pri_queue_iter_next (&iter, &elem)) + ELEM_TO_ITEM (elem)->inserted += 1; + + for (i = 0; i < NUM_ITEMS; i++) + fail_unless_equals_int (1, items[i].inserted); + + gst_pri_queue_destroy (pq, NULL); + g_free (items); +} + +GST_END_TEST; + +static Suite * +gst_pri_queue_suite (void) +{ + Suite *s = suite_create ("GstPriQueue"); + TCase *tc_chain = tcase_create ("general"); + + suite_add_tcase (s, tc_chain); + + tcase_add_test (tc_chain, test_random_modifications); + tcase_add_test (tc_chain, test_sorted_insertion); + tcase_add_test (tc_chain, test_sorted_removal); + tcase_add_test (tc_chain, test_pop_min); + tcase_add_test (tc_chain, test_destroy); + tcase_add_test (tc_chain, test_meld); + tcase_add_test (tc_chain, test_iter); + + return s; +} + +GST_CHECK_MAIN (gst_pri_queue); From b2e0ce90975359ab1515796ea3ec8918bd45b761 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sun, 3 May 2020 13:08:37 +0200 Subject: [PATCH 103/339] gstaggregator: downgrade latency-warning to info --- subprojects/gstreamer/libs/gst/base/gstaggregator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gstreamer/libs/gst/base/gstaggregator.c b/subprojects/gstreamer/libs/gst/base/gstaggregator.c index a6aa43af0cb..833f7ac847c 100644 --- a/subprojects/gstreamer/libs/gst/base/gstaggregator.c +++ b/subprojects/gstreamer/libs/gst/base/gstaggregator.c @@ -2267,7 +2267,7 @@ gst_aggregator_query_latency_unlocked (GstAggregator * self, GstQuery * query) SRC_LOCK (self); if (!query_ret) { - GST_WARNING_OBJECT (self, "Latency query failed"); + GST_INFO_OBJECT (self, "Latency query failed"); return FALSE; } From 509851f157de1b94e0662c8990dee7fdb8036a9a Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 4 May 2020 12:30:15 +0200 Subject: [PATCH 104/339] systemclock: add conditional include for pthread.h This is needed to compile for iOS 64bit with XCode 11.3.1 --- subprojects/gstreamer/gst/gstsystemclock.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/subprojects/gstreamer/gst/gstsystemclock.c b/subprojects/gstreamer/gst/gstsystemclock.c index 827a7e1b565..fcc85831210 100644 --- a/subprojects/gstreamer/gst/gstsystemclock.c +++ b/subprojects/gstreamer/gst/gstsystemclock.c @@ -47,6 +47,10 @@ #include +#ifdef HAVE_PTHREAD_H +#include +#endif + #ifdef G_OS_WIN32 # define WIN32_LEAN_AND_MEAN /* prevents from including too many things */ # include /* QueryPerformance* stuff */ From db1742557186d76ae6734a30cbb338d960ee8ea3 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 29 May 2020 11:31:04 -0300 Subject: [PATCH 105/339] gstdeviceprovider: added cleanup of hidden providers list --- subprojects/gstreamer/gst/gstdeviceprovider.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/subprojects/gstreamer/gst/gstdeviceprovider.c b/subprojects/gstreamer/gst/gstdeviceprovider.c index f073ede51b7..1699047e929 100644 --- a/subprojects/gstreamer/gst/gstdeviceprovider.c +++ b/subprojects/gstreamer/gst/gstdeviceprovider.c @@ -180,8 +180,13 @@ gst_device_provider_dispose (GObject * object) gst_object_replace ((GstObject **) & provider->priv->bus, NULL); GST_OBJECT_LOCK (provider); + g_list_free_full (provider->devices, (GDestroyNotify) gst_object_unparent); provider->devices = NULL; + + g_list_free_full (provider->priv->hidden_providers, (GDestroyNotify) g_free); + provider->priv->hidden_providers = NULL; + GST_OBJECT_UNLOCK (provider); G_OBJECT_CLASS (gst_device_provider_parent_class)->dispose (object); From 7ff541c67f816c7182b750aca2ca3d5e2c89802a Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Wed, 17 Jun 2020 13:54:05 +0200 Subject: [PATCH 106/339] gstpad: warn not linked flow errors [pexhack] --- subprojects/gstreamer/gst/gstghostpad.c | 8 ++++++-- subprojects/gstreamer/gst/gstpad.c | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/subprojects/gstreamer/gst/gstghostpad.c b/subprojects/gstreamer/gst/gstghostpad.c index 99f1cf53e39..6041031b4a4 100644 --- a/subprojects/gstreamer/gst/gstghostpad.c +++ b/subprojects/gstreamer/gst/gstghostpad.c @@ -59,8 +59,12 @@ #define GST_PROXY_PAD_ACQUIRE_INTERNAL(pad, internal, retval) \ internal = \ GST_PAD_CAST (gst_proxy_pad_get_internal (GST_PROXY_PAD_CAST (pad))); \ - if (internal == NULL) \ - return retval; + if (internal == NULL) {\ + if ((GstFlowReturn)retval == GST_FLOW_NOT_LINKED) {\ + GST_WARNING_OBJECT ((pad), "%p:%p returning not-linked", GST_PAD_PARENT(pad), (pad));\ + }\ + return retval;\ + } #define GST_PROXY_PAD_RELEASE_INTERNAL(internal) gst_object_unref (internal); diff --git a/subprojects/gstreamer/gst/gstpad.c b/subprojects/gstreamer/gst/gstpad.c index 2dc9c0376c4..2e661356349 100644 --- a/subprojects/gstreamer/gst/gstpad.c +++ b/subprojects/gstreamer/gst/gstpad.c @@ -150,6 +150,8 @@ struct _GstPadPrivate * by a single thread at a time. Protected by the object lock */ GCond activation_cond; gboolean in_activation; + + gboolean warned_unlinked; }; typedef struct @@ -223,6 +225,18 @@ static GstFlowQuarks flow_quarks[] = { {GST_FLOW_CUSTOM_ERROR, "custom-error", 0} }; +static void +gst_pad_warn_if_unlinked (GstPad *pad, GstFlowReturn res) { + if (G_UNLIKELY (GST_PAD_PEER (pad) == NULL && res == GST_FLOW_NOT_LINKED)) { + if (!pad->priv->warned_unlinked) { + pad->priv->warned_unlinked = TRUE; + GST_WARNING_OBJECT (pad, "%p:%p returning not-linked", GST_PAD_PARENT(pad), pad); + } + } else { + pad->priv->warned_unlinked = FALSE; + } +} + /** * gst_flow_get_name: * @ret: a #GstFlowReturn to get the name of. @@ -417,6 +431,7 @@ gst_pad_init (GstPad * pad) pad->priv->events = g_array_sized_new (FALSE, TRUE, sizeof (PadEvent), 16); pad->priv->events_cookie = 0; pad->priv->last_cookie = -1; + pad->priv->warned_unlinked = FALSE; g_cond_init (&pad->priv->activation_cond); pad->ABI.abi.last_flowret = GST_FLOW_FLUSHING; @@ -4860,6 +4875,7 @@ gst_pad_push (GstPad * pad, GstBuffer * buffer) GST_TRACER_PAD_PUSH_PRE (pad, buffer); res = gst_pad_push_data (pad, GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_PUSH, buffer); + gst_pad_warn_if_unlinked (pad, res); GST_TRACER_PAD_PUSH_POST (pad, res); return res; } @@ -4900,6 +4916,7 @@ gst_pad_push_list (GstPad * pad, GstBufferList * list) GST_TRACER_PAD_PUSH_LIST_PRE (pad, list); res = gst_pad_push_data (pad, GST_PAD_PROBE_TYPE_BUFFER_LIST | GST_PAD_PROBE_TYPE_PUSH, list); + gst_pad_warn_if_unlinked (pad, res); GST_TRACER_PAD_PUSH_LIST_POST (pad, res); return res; } @@ -5296,6 +5313,7 @@ gst_pad_pull_range (GstPad * pad, guint64 offset, guint size, } done: GST_TRACER_PAD_PULL_RANGE_POST (pad, NULL, ret); + gst_pad_warn_if_unlinked (pad, ret); return ret; } From 3fee370edd2335baf69b56eb4356aef848890782 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 23 Mar 2021 15:20:30 +0100 Subject: [PATCH 107/339] funnel: temporary workaround for lost-events triggering sticky resend We want a better more permanent patch here, but this is the smallest conceivable fix for a dot-release. --- .../gstreamer/plugins/elements/gstfunnel.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/subprojects/gstreamer/plugins/elements/gstfunnel.c b/subprojects/gstreamer/plugins/elements/gstfunnel.c index b09dbf18d11..9be847fc5b6 100644 --- a/subprojects/gstreamer/plugins/elements/gstfunnel.c +++ b/subprojects/gstreamer/plugins/elements/gstfunnel.c @@ -361,6 +361,16 @@ gst_funnel_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GST_MINI_OBJECT_CAST (buffer)); } +static gboolean +_is_lost_event (GstEvent * event) +{ + const GstStructure *s; + s = gst_event_get_structure (event); + if (s) + return gst_structure_has_name (s, "GstRTPPacketLost"); + return FALSE; +} + static gboolean gst_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -369,6 +379,7 @@ gst_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) gboolean forward = TRUE; gboolean res = TRUE; gboolean unlock = FALSE; + gboolean is_lost_event; GST_DEBUG_OBJECT (pad, "received event %" GST_PTR_FORMAT, event); @@ -396,7 +407,9 @@ gst_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) GST_OBJECT_UNLOCK (funnel); } - if (forward && GST_EVENT_IS_SERIALIZED (event)) { + /* FIXME: make something better around lost-events. rtpfunnel? */ + is_lost_event = _is_lost_event (event); + if (!is_lost_event && forward && GST_EVENT_IS_SERIALIZED (event)) { /* If no data is coming and we receive serialized event, need to forward all sticky events. * Otherwise downstream has an inconsistent set of sticky events when * handling the new event. */ From 3bce75add071c636c89a23729c0b26ef3d44574f Mon Sep 17 00:00:00 2001 From: Will Miller Date: Mon, 26 Apr 2021 20:59:55 +0100 Subject: [PATCH 108/339] helpers/gst_gdb: fixes to printing of GstStructures - Don't crash if no fundamental type name is found for a GValue - Print the fundamental type of each GValue in a GstStructure - Fix type name comparison to GEnum --- subprojects/gstreamer/libs/gst/helpers/gst_gdb.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py b/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py index 90374fba976..f8273e27964 100644 --- a/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py +++ b/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py @@ -423,7 +423,9 @@ def __str__(self): value = self.value() tname = self.fundamental_typename() gvalue_type = gdb.lookup_type("GValue") - if tname == "GstFraction": + if tname is None: + v = "" + elif tname == "GstFraction": v = "%d/%d" % (value[0], value[1]) elif tname == "GstBitmask": v = "0x%016x" % long(value) @@ -450,7 +452,7 @@ def __str__(self): v += " " if v == "<" else ", " v += str(GdbGValue(l)) v += " >" - elif tname in ("GEnum"): + elif tname == "GEnum": v = "%s(%s)" % ( g_type_to_name(g_type_to_typenode(self.val["g_type"])), value["v_int"]) @@ -527,7 +529,7 @@ def print(self, indent, prefix=None): else: _gdb_write(indent, "%s:" % (self.name())) for (key, value) in self.values(): - _gdb_write(indent+1, "%s: %s" % (key, str(value))) + _gdb_write(indent+1, "%s: %s (type %s)" % (key, str(value), value.fundamental_typename())) class GdbGstSegment: From f775289427f165b70a68e98a60fd64fa2a47d1ed Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 30 Apr 2021 13:09:04 +0200 Subject: [PATCH 109/339] tests/element: test to show releasing request-pads is unsafe Co-authored-by: Will Miller --- .../gstreamer/tests/check/gst/gstelement.c | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/subprojects/gstreamer/tests/check/gst/gstelement.c b/subprojects/gstreamer/tests/check/gst/gstelement.c index a80aa45e675..a0e07f5199a 100644 --- a/subprojects/gstreamer/tests/check/gst/gstelement.c +++ b/subprojects/gstreamer/tests/check/gst/gstelement.c @@ -962,6 +962,62 @@ GST_START_TEST (test_foreach_pad) GST_END_TEST; +typedef struct +{ + GstPad *srcpads[1000]; +} ReleaseDisposeCtx; + +static void +_release_request_pad (GstPad * pad) +{ + GstElement *parent; + if (pad == NULL) + return; + parent = GST_ELEMENT (gst_pad_get_parent (pad)); + if (parent) { + gst_element_release_request_pad (parent, pad); + gst_object_unref (parent); + } + gst_object_unref (pad); +} + +static gpointer +_release_pad_func (gpointer user_data) +{ + ReleaseDisposeCtx *ctx = user_data; + for (guint i = 0; i < G_N_ELEMENTS (ctx->srcpads); i++) + _release_request_pad (ctx->srcpads[i]); + return NULL; +} + +GST_START_TEST (test_release_pads_during_dispose) +{ + GstElement *pipeline = gst_pipeline_new (NULL); + GstTestElement3 *te3 = g_object_new (gst_test_element3_get_type (), NULL); + GstElement *element = GST_ELEMENT_CAST (te3); + ReleaseDisposeCtx ctx; + GThread *query_thread; + + gst_bin_add (GST_BIN (pipeline), element); + + /* request lots of pads */ + for (guint i = 0; i < G_N_ELEMENTS (ctx.srcpads); i++) { + ctx.srcpads[i] = gst_element_get_request_pad (element, "src_%d"); + } + + /* start a thread to start releasing those pads */ + query_thread = g_thread_new (NULL, _release_pad_func, &ctx); + + /* while at the same time, shutting down the pipeline */ + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); + + g_thread_join (query_thread); +} + +GST_END_TEST; + + static Suite * gst_element_suite (void) { @@ -980,6 +1036,7 @@ gst_element_suite (void) tcase_add_test (tc_chain, test_request_pad_templates); tcase_add_test (tc_chain, test_forbidden_pad_template_names); tcase_add_test (tc_chain, test_foreach_pad); + tcase_add_test (tc_chain, test_release_pads_during_dispose); return s; } From 2ab11250022ecc624f266487fe32679b3ca30bc9 Mon Sep 17 00:00:00 2001 From: Will Miller Date: Wed, 16 Jun 2021 14:15:31 +0100 Subject: [PATCH 110/339] .gitignore: ignore __pycache__ directory This is created when using the gst-gdb Python helper script. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7143eefa90d..12d7f55d162 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ subprojects/*/ # clangd integration .cache/* + +# from gdb Python helper scripts +__pycache__ From 0e368a17919997b53665739845a073846701d66d Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 15 Sep 2021 18:12:14 +0200 Subject: [PATCH 111/339] gst: don't error out on already deinit GStreamer Some python-bindings are doing this twice, so we need to be resilient. --- subprojects/gstreamer/gst/gst.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/subprojects/gstreamer/gst/gst.c b/subprojects/gstreamer/gst/gst.c index 3f9096197ba..a99a678f6b3 100644 --- a/subprojects/gstreamer/gst/gst.c +++ b/subprojects/gstreamer/gst/gst.c @@ -1088,7 +1088,9 @@ gst_deinit (void) } if (gst_deinitialized) { /* tell the user how naughty they've been */ - g_error ("GStreamer should not be deinitialized a second time."); + g_message ("GStreamer should not be deinitialized a second time."); + g_mutex_unlock (&init_lock); + return; } GST_INFO ("deinitializing GStreamer"); From 3998eee951024f46a4c033b9369cb9cb11e9a230 Mon Sep 17 00:00:00 2001 From: Frederik Vestre Date: Fri, 20 Aug 2021 14:10:56 +0200 Subject: [PATCH 112/339] Use refcounting for gstclockid instead of weakref --- subprojects/gstreamer/gst/gst_private.h | 2 +- subprojects/gstreamer/gst/gstclock.c | 43 +++++++++++++------------ 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/subprojects/gstreamer/gst/gst_private.h b/subprojects/gstreamer/gst/gst_private.h index 9be91ef96e4..0c7a01dc922 100644 --- a/subprojects/gstreamer/gst/gst_private.h +++ b/subprojects/gstreamer/gst/gst_private.h @@ -512,7 +512,7 @@ struct _GstDynamicTypeFactoryClass { struct _GstClockEntryImpl { GstClockEntry entry; - GWeakRef *clock; + GstClock *clock; GDestroyNotify destroy_entry; gpointer padding[21]; /* padding for allowing e.g. systemclock * to add data in lieu of overridable diff --git a/subprojects/gstreamer/gst/gstclock.c b/subprojects/gstreamer/gst/gstclock.c index ab8a4b00255..1ef3a615fbf 100644 --- a/subprojects/gstreamer/gst/gstclock.c +++ b/subprojects/gstreamer/gst/gstclock.c @@ -134,6 +134,9 @@ enum struct _GstClockPrivate { + /* Not locked, but only atomic use */ + gint weak_refcount; + GMutex slave_lock; /* order: SLAVE_LOCK, OBJECT_LOCK */ GCond sync_cond; @@ -171,7 +174,7 @@ struct _GstClockPrivate typedef struct _GstClockEntryImpl GstClockEntryImpl; -#define GST_CLOCK_ENTRY_CLOCK_WEAK_REF(entry) (((GstClockEntryImpl *)(entry))->clock) +#define GST_CLOCK_ENTRY_CLOCK_GET(entry) (((GstClockEntryImpl *)(entry))->clock) /* seqlocks */ #define read_seqbegin(clock) \ @@ -262,8 +265,8 @@ gst_clock_entry_new (GstClock * clock, GstClockTime time, entry->_clock = clock; #endif #endif - GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry) = - g_atomic_rc_box_acquire (clock->priv->clock_weakref); + GST_CLOCK_ENTRY_CLOCK_GET (entry) = clock; + g_atomic_int_inc (&(GST_CLOCK_ENTRY_CLOCK_GET (entry)->priv->weak_refcount)); entry->type = type; entry->time = time; entry->interval = interval; @@ -377,9 +380,10 @@ _gst_clock_id_free (GstClockID id) if (entry_impl->destroy_entry) entry_impl->destroy_entry (entry_impl); - g_atomic_rc_box_release_full (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry), - (GDestroyNotify) g_weak_ref_clear); - + gboolean weak_leak = + g_atomic_int_dec_and_test (&(GST_CLOCK_ENTRY_CLOCK_GET (entry)-> + priv->weak_refcount)); + g_assert (!weak_leak); /* FIXME: add tracer hook for struct allocations such as clock entries */ g_free (id); @@ -535,7 +539,7 @@ gst_clock_id_wait (GstClockID id, GstClockTimeDiff * jitter) entry = (GstClockEntry *) id; requested = GST_CLOCK_ENTRY_TIME (entry); - clock = g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + clock = GST_CLOCK_ENTRY_CLOCK_GET (entry); if (G_UNLIKELY (clock == NULL)) goto invalid_entry; @@ -560,7 +564,6 @@ gst_clock_id_wait (GstClockID id, GstClockTimeDiff * jitter) if (entry->type == GST_CLOCK_ENTRY_PERIODIC) entry->time = requested + entry->interval; - gst_object_unref (clock); return res; /* ERRORS */ @@ -568,13 +571,11 @@ gst_clock_id_wait (GstClockID id, GstClockTimeDiff * jitter) { GST_CAT_DEBUG_OBJECT (GST_CAT_CLOCK, clock, "invalid time requested, returning _BADTIME"); - gst_object_unref (clock); return GST_CLOCK_BADTIME; } not_supported: { GST_CAT_DEBUG_OBJECT (GST_CAT_CLOCK, clock, "clock wait is not supported"); - gst_object_unref (clock); return GST_CLOCK_UNSUPPORTED; } invalid_entry: @@ -617,7 +618,7 @@ gst_clock_id_wait_async (GstClockID id, entry = (GstClockEntry *) id; requested = GST_CLOCK_ENTRY_TIME (entry); - clock = g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + clock = GST_CLOCK_ENTRY_CLOCK_GET (entry); if (G_UNLIKELY (clock == NULL)) goto invalid_entry; @@ -636,7 +637,6 @@ gst_clock_id_wait_async (GstClockID id, res = cclass->wait_async (clock, entry); - gst_object_unref (clock); return res; /* ERRORS */ @@ -645,13 +645,11 @@ gst_clock_id_wait_async (GstClockID id, (func) (clock, GST_CLOCK_TIME_NONE, id, user_data); GST_CAT_DEBUG_OBJECT (GST_CAT_CLOCK, clock, "invalid time requested, returning _BADTIME"); - gst_object_unref (clock); return GST_CLOCK_BADTIME; } not_supported: { GST_CAT_DEBUG_OBJECT (GST_CAT_CLOCK, clock, "clock wait is not supported"); - gst_object_unref (clock); return GST_CLOCK_UNSUPPORTED; } invalid_entry: @@ -680,7 +678,7 @@ gst_clock_id_unschedule (GstClockID id) g_return_if_fail (id != NULL); entry = (GstClockEntry *) id; - clock = g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + clock = GST_CLOCK_ENTRY_CLOCK_GET (entry); if (G_UNLIKELY (clock == NULL)) goto invalid_entry; @@ -689,7 +687,6 @@ gst_clock_id_unschedule (GstClockID id) if (G_LIKELY (cclass->unschedule)) cclass->unschedule (clock, entry); - gst_object_unref (clock); return; invalid_entry: @@ -757,6 +754,8 @@ gst_clock_init (GstClock * clock) clock->priv = priv = gst_clock_get_instance_private (clock); + priv->weak_refcount = 1; /* Set it to 1 so we can assert on dec and test */ + priv->last_time = 0; priv->internal_calibration = 0; @@ -793,6 +792,11 @@ gst_clock_dispose (GObject * object) GstClock **master_p; GST_OBJECT_LOCK (clock); + + /* Ensure we have no dangeling weak pointers */ + int num_weak_refs = g_atomic_int_get (&(clock->priv->weak_refcount)); + g_assert (num_weak_refs == 1); + master_p = &clock->priv->master; gst_object_replace ((GstObject **) master_p, NULL); GST_OBJECT_UNLOCK (clock); @@ -1391,7 +1395,7 @@ gst_clock_id_get_clock (GstClockID id) g_return_val_if_fail (id != NULL, NULL); entry = (GstClockEntry *) id; - return g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + return gst_object_ref (GST_CLOCK_ENTRY_CLOCK_GET (entry)); } /** @@ -1418,13 +1422,10 @@ gst_clock_id_uses_clock (GstClockID id, GstClock * clock) g_return_val_if_fail (clock != NULL, FALSE); entry = (GstClockEntry *) id; - entry_clock = g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + entry_clock = GST_CLOCK_ENTRY_CLOCK_GET (entry); if (entry_clock == clock) ret = TRUE; - if (G_LIKELY (entry_clock != NULL)) - gst_object_unref (entry_clock); - return ret; } From 8684d3d023f973158ec4784bc2f6530af3890348 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Sat, 11 Sep 2021 23:23:59 +0900 Subject: [PATCH 113/339] check: add missing suppresion for debug function list We also leak during removal of a function --- subprojects/gstreamer/tests/check/gstreamer.supp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/subprojects/gstreamer/tests/check/gstreamer.supp b/subprojects/gstreamer/tests/check/gstreamer.supp index e69c0cfb7be..3729dfd2b43 100644 --- a/subprojects/gstreamer/tests/check/gstreamer.supp +++ b/subprojects/gstreamer/tests/check/gstreamer.supp @@ -4035,6 +4035,15 @@ fun:gst_debug_add_log_function } +{ + Leak of debug function list + Memcheck:Leak + fun:*alloc + ... + fun:g_slist_copy_deep + fun:gst_debug_remove_with_compare_func +} + { Leak of debug function list item Memcheck:Leak From 03cec0b120d44e20b1e0e8a34562c1394ac7f2cf Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 14 Jul 2015 14:37:16 +0200 Subject: [PATCH 114/339] rtpbasepayload: set DEFAULT_PERFECT_RTPTIME to FALSE A very common pipeline is feeding an encoder inheriting from GstAudioEncoder into a payloader inheriting from GstRtpBasePayload. If this is the case, the rtp-timestamp will, if perfect_rtptime is TRUE, reflect the amount of bytes in each packet, and not (like it should) reflect the timestamp of the packets in the clock-rate domain. Example: 20ms AAC packets with clock-rate of 90000, should have the rtp-timestamp incrementing with 90000 * 20 / 1000 = 1800. Currently, because perfect_rtptime is default TRUE, the AAC-packets will be incrementing with the buffer-size of the packets, which is typically 64000 * 20 / 1000 = 1280. In the case that bitrate and clock-rate are the same (like with ALAW and MULAW) it will work by chance, which is probably why it has not been discovered earlier. https://bugzilla.gnome.org/show_bug.cgi?id=761943 --- .../gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c index 02d0f51d234..17261827e4d 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c @@ -105,7 +105,7 @@ static guint gst_rtp_base_payload_signals[LAST_SIGNAL] = { 0 }; #define DEFAULT_SEQNUM_OFFSET -1 #define DEFAULT_MAX_PTIME -1 #define DEFAULT_MIN_PTIME 0 -#define DEFAULT_PERFECT_RTPTIME TRUE +#define DEFAULT_PERFECT_RTPTIME FALSE #define DEFAULT_PTIME_MULTIPLE 0 #define DEFAULT_RUNNING_TIME GST_CLOCK_TIME_NONE #define DEFAULT_SOURCE_INFO FALSE From 5d68069cd658088bf86a46fa15b80549ed48c2b0 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 14 Mar 2016 15:45:49 +0100 Subject: [PATCH 115/339] audiodecoder: Degrade warnings When doing DTX generation in decoders, they are indeed producing buffers without having a corresponding input buffer, so this code needs to be revisited, and perhaps need a special method for pushing buffers like this? https://bugzilla.gnome.org/show_bug.cgi?id=773106 --- .../gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c index acb6314ea07..a88d76990cb 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c @@ -1416,9 +1416,9 @@ gst_audio_decoder_finish_frame_or_subframe (GstAudioDecoder * dec, } } else if (G_UNLIKELY (frames > priv->frames.length)) { if (G_LIKELY (!priv->force)) { - GST_ELEMENT_WARNING (dec, STREAM, DECODE, - ("received more decoded frames %d than provided %d", frames, - priv->frames.length), (NULL)); + GST_DEBUG_OBJECT (dec, + "received more decoded frames %d than provided %d", frames, + priv->frames.length); } frames = priv->frames.length; } @@ -1536,7 +1536,7 @@ gst_audio_decoder_finish_frame_or_subframe (GstAudioDecoder * dec, data.outbuf = buf; gst_buffer_foreach_meta (in_buf, foreach_metadata, &data); } else { - GST_WARNING_OBJECT (dec, + GST_DEBUG_OBJECT (dec, "Can't copy metadata because input buffers disappeared"); } } From ccd66b7cde48783371f6607d21c9ad5cbeb97875 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Tue, 2 Feb 2016 13:06:23 +0100 Subject: [PATCH 116/339] Revert "videopool: ensure allocation alignment is consistent with video alignment requirements" [pexhack] This reverts commit 8b96b52a62567d70ce827db00fd0f50f79133ee5. --- .../gst-libs/gst/video/gstvideopool.c | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideopool.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideopool.c index fd4198d79f8..ccbbff42be7 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideopool.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideopool.c @@ -170,31 +170,14 @@ video_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config) GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); if (priv->need_alignment && priv->add_videometa) { - guint max_align, n; - + /* get and apply the alignment to the info */ gst_buffer_pool_config_get_video_alignment (config, &priv->video_align); - /* ensure GstAllocationParams alignment is compatible with video alignment */ - max_align = priv->params.align; - for (n = 0; n < GST_VIDEO_MAX_PLANES; ++n) - max_align |= priv->video_align.stride_align[n]; - - for (n = 0; n < GST_VIDEO_MAX_PLANES; ++n) - priv->video_align.stride_align[n] = max_align; - /* apply the alignment to the info */ if (!gst_video_info_align (&info, &priv->video_align)) goto failed_to_align; gst_buffer_pool_config_set_video_alignment (config, &priv->video_align); - - if (priv->params.align < max_align) { - GST_WARNING_OBJECT (pool, "allocation params alignment %u is smaller " - "than the max specified video stride alignment %u, fixing", - (guint) priv->params.align, max_align); - priv->params.align = max_align; - gst_buffer_pool_config_set_allocator (config, allocator, &priv->params); - } } info.size = MAX (size, info.size); priv->info = info; From 331e34f9e2b0cc22351ddd61e93c7e5d2840eaac Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 4 Feb 2016 10:36:05 +0100 Subject: [PATCH 117/339] Revert "Revert "audioencoder: timestamp headers same as first buffer and use duration 0"" [pexhack] This reverts commit b60ab758e4ba3b6346542c4c8a004f0a51732709. https://bugzilla.gnome.org/show_bug.cgi?id=754224 --- .../gst-libs/gst/audio/gstaudioencoder.c | 22 ++++++++-- .../tests/check/libs/audioencoder.c | 42 +++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudioencoder.c b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudioencoder.c index f77216709bd..0ccf0bde64e 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudioencoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudioencoder.c @@ -918,6 +918,16 @@ gst_audio_encoder_finish_frame (GstAudioEncoder * enc, GstBuffer * buf, /* collect output */ if (G_LIKELY (buf)) { gsize size; + GstClockTime timestamp = GST_CLOCK_TIME_NONE; + + if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (priv->base_ts))) { + /* FIXME ? lookahead could lead to weird ts and duration ? + * (particularly if not in perfect mode) */ + /* mind sample rounding and produce perfect output */ + timestamp = priv->base_ts + + gst_util_uint64_scale (priv->samples - ctx->lookahead, GST_SECOND, + ctx->info.rate); + } /* Pushing headers first */ if (G_UNLIKELY (priv->ctx.new_headers)) { @@ -931,6 +941,12 @@ gst_audio_encoder_finish_frame (GstAudioEncoder * enc, GstBuffer * buf, tmpbuf = gst_buffer_make_writable (tmpbuf); size = gst_buffer_get_size (tmpbuf); + /* we timestamp the headers with the same timestamp as the first + buffer, and set their duration to 0 */ + GST_BUFFER_PTS (tmpbuf) = timestamp; + GST_BUFFER_DTS (tmpbuf) = timestamp; + GST_BUFFER_DURATION (tmpbuf) = 0; + if (G_UNLIKELY (priv->discont)) { GST_LOG_OBJECT (enc, "marking discont"); GST_BUFFER_FLAG_SET (tmpbuf, GST_BUFFER_FLAG_DISCONT); @@ -971,10 +987,8 @@ gst_audio_encoder_finish_frame (GstAudioEncoder * enc, GstBuffer * buf, /* FIXME ? lookahead could lead to weird ts and duration ? * (particularly if not in perfect mode) */ /* mind sample rounding and produce perfect output */ - GST_BUFFER_PTS (buf) = priv->base_ts + - gst_util_uint64_scale (priv->samples - ctx->lookahead, GST_SECOND, - ctx->info.rate); - GST_BUFFER_DTS (buf) = GST_BUFFER_PTS (buf); + GST_BUFFER_PTS (buf) = timestamp; + GST_BUFFER_DTS (buf) = timestamp; GST_DEBUG_OBJECT (enc, "out samples %d", samples); if (G_LIKELY (samples > 0)) { priv->samples += samples; diff --git a/subprojects/gst-plugins-base/tests/check/libs/audioencoder.c b/subprojects/gst-plugins-base/tests/check/libs/audioencoder.c index bcaf8d98b8c..93ece17cbbf 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/audioencoder.c +++ b/subprojects/gst-plugins-base/tests/check/libs/audioencoder.c @@ -395,6 +395,46 @@ GST_START_TEST (audioencoder_events_before_eos) GST_END_TEST; +GST_START_TEST (audioencoder_headers_timestamp) +{ + GstHarness *h = setup_audioencodertester (); + GList *headers = NULL; + GstBuffer *buf; + + /* create and add headers */ + headers = g_list_append (headers, gst_buffer_new ()); + headers = g_list_append (headers, gst_buffer_new ()); + gst_audio_encoder_set_headers (GST_AUDIO_ENCODER (h->element), headers); + + /* push buffer with timestamp 10 seconds*/ + fail_unless (gst_harness_push (h, create_test_buffer (10)) == GST_FLOW_OK); + + /* first header */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_DTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* second header */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_DTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* buffer */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_DTS (buf)); + fail_unless_equals_uint64 (1 * GST_SECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} + +GST_END_TEST; + static Suite * gst_audioencoder_suite (void) { @@ -408,6 +448,8 @@ gst_audioencoder_suite (void) tcase_add_test (tc, audioencoder_events_before_eos); tcase_add_test (tc, audioencoder_flush_events); + tcase_add_test (tc, audioencoder_headers_timestamp); + return s; } From af9261404d5c4faca678eb679b7faaa0f6bfee58 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 28 Jul 2014 12:07:50 +0200 Subject: [PATCH 118/339] rtppayloads: Change dynamic payload type to [0,127] [pexhack] Depayloaders should be less strict about spec conformance on incoming packets. --- subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h index 84f6c3cc3cc..98d9f288701 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h @@ -149,7 +149,7 @@ typedef enum #define GST_RTP_PAYLOAD_MP2T_STRING "33" #define GST_RTP_PAYLOAD_H263_STRING "34" -#define GST_RTP_PAYLOAD_DYNAMIC_STRING "[96, 127]" +#define GST_RTP_PAYLOAD_DYNAMIC_STRING "[0, 127]" /** * GST_RTP_PAYLOAD_IS_DYNAMIC: From b5d6408a09d1520f6c096d2f73f51937c31cffb4 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 19 Jun 2017 00:18:18 +0200 Subject: [PATCH 119/339] audiodecoder: don't ask for concealment for 0-duration gaps --- .../gst-libs/gst/audio/gstaudiodecoder.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c index a88d76990cb..8d9357af22e 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c @@ -2294,12 +2294,16 @@ gst_audio_decoder_handle_gap (GstAudioDecoder * dec, GstEvent * event) GstAudioDecoderClass *klass = GST_AUDIO_DECODER_GET_CLASS (dec); GstBuffer *buf; - /* hand subclass empty frame with duration that needs covering */ - buf = gst_buffer_new (); - GST_BUFFER_PTS (buf) = timestamp; - GST_BUFFER_DURATION (buf) = duration; - /* best effort, not much error handling */ - gst_audio_decoder_handle_frame (dec, klass, buf); + /* asking to conceal 0 duration does not make sense, specially since + _finish_frame is picky about the buffer there actually having a size */ + if (duration > 0) { + /* hand subclass empty frame with duration that needs covering */ + buf = gst_buffer_new (); + GST_BUFFER_PTS (buf) = timestamp; + GST_BUFFER_DURATION (buf) = duration; + /* best effort, not much error handling */ + gst_audio_decoder_handle_frame (dec, klass, buf); + } ret = TRUE; dec->priv->expecting_discont_buf = TRUE; gst_event_unref (event); From de9f8151c96df9fb6994f974da248c3885efe331 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Mon, 21 Jan 2019 14:20:15 +0100 Subject: [PATCH 120/339] [pexhack] videorate: enable drop-only when possible With this patch https://github.com/pexip/mcu/commit/f819d921773ab222ee0fb950e9808c48b3794156 @johnbassett discovered that old frames would be produced if stopping and starting a layer. However, setting drop-only always to true means we can never increase the framerate making the caps negotiation fail. This hack sets drop-only to true when the input rate is higher or equal to the output one. A proper fix would be to identify the bug @johnbassett discovered and do a test and fix for that in the videorate element. --- .../gst-plugins-base/gst/videorate/gstvideorate.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c b/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c index f16b4753838..44bc68166a3 100644 --- a/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c +++ b/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c @@ -613,6 +613,16 @@ gst_video_rate_setcaps (GstBaseTransform * trans, GstCaps * in_caps, else videorate->wanted_diff = 0; + /* PEXHACK + * drop-only needed due to: + * https://github.com/pexip/mcu/commit/f819d921773ab222ee0fb950e9808c48b3794156 + */ + { + gint in_fps = (videorate->from_rate_numerator * 1000) / videorate->from_rate_denominator; + gint out_fps = (videorate->to_rate_numerator * 1000) / videorate->to_rate_denominator; + if (in_fps >= out_fps) + videorate->drop_only = TRUE; + } done: if (ret) { gst_caps_replace (&videorate->in_caps, in_caps); From 309a3cd334984dcb4a827e0d0a7e81a5046d0f72 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 17 Sep 2019 08:42:52 +0200 Subject: [PATCH 121/339] [pexhack] gst-libs/gl/meson: disable -fobjc-arc check for now [pexhack] --- subprojects/gst-plugins-base/ext/gl/meson.build | 6 +++--- subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-base/ext/gl/meson.build b/subprojects/gst-plugins-base/ext/gl/meson.build index ad514014e9a..a5b08fb7a65 100644 --- a/subprojects/gst-plugins-base/ext/gl/meson.build +++ b/subprojects/gst-plugins-base/ext/gl/meson.build @@ -124,9 +124,9 @@ if ['darwin', 'ios'].contains(host_system) endif objc = meson.get_compiler('objc') - if not objc.has_argument('-fobjc-arc') - error('ARC is required for building') - endif +# if not objc.has_argument('-fobjc-arc') +# error('ARC is required for building') +# endif gl_objc_args += ['-fobjc-arc'] endif diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build b/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build index ebdab9089ab..ad3bd339d33 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build @@ -800,9 +800,9 @@ if ['darwin', 'ios'].contains(host_system) endif objc = meson.get_compiler('objc') - if not objc.has_argument('-fobjc-arc') - error('ARC is required for building') - endif + #if not objc.has_argument('-fobjc-arc') + # error('ARC is required for building') + #endif gl_objc_args += ['-fobjc-arc'] endif From 9b4769ddeb566855c95c06d86d581fb23eb955a7 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 28 Jan 2020 11:37:39 +0100 Subject: [PATCH 122/339] [pexhack] Revert "videoencoder: Use video marker flag to signal end of frame" This is because we set this flag ourselves in the implementations. This reverts commit 7013a5887402c6b8a0689bd0824d923a7573c16a. --- .../gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c index 2c269e1aa3b..d0efd5a1758 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c @@ -2579,12 +2579,6 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, GST_BUFFER_DTS (frame->output_buffer) = frame->dts; GST_BUFFER_DURATION (frame->output_buffer) = frame->duration; - /* At this stage we have a full frame in subframe use case , - * let's mark it to enabled some latency optimization - * in some uses cases like RTP. */ - - GST_BUFFER_FLAG_SET (frame->output_buffer, GST_VIDEO_BUFFER_FLAG_MARKER); - GST_OBJECT_LOCK (encoder); /* update rate estimate */ priv->bytes += gst_buffer_get_size (frame->output_buffer); From ac21e71f41436844ebf9c2e36bed00344c6f9652 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Fri, 26 Mar 2021 22:08:38 +0900 Subject: [PATCH 123/339] [pexhack] rtp/test: temporarly comment out ASSERT_CRITICAL See https://github.com/pexip/mcu/issues/21304 --- subprojects/gst-plugins-base/tests/check/libs/rtp.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtp.c b/subprojects/gst-plugins-base/tests/check/libs/rtp.c index 1fd25f3d6b2..4771f351b38 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtp.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtp.c @@ -110,7 +110,9 @@ GST_START_TEST (test_rtp_buffer) /* check csrc bits */ fail_unless_equals_int (gst_rtp_buffer_get_csrc_count (&rtp), 0); - ASSERT_CRITICAL (gst_rtp_buffer_get_csrc (&rtp, 0)); + GST_FIXME + ("Unable to check CSRC. ASSERT_CRITICAL is broken. See MCU issue 21304"); + // ASSERT_CRITICAL (gst_rtp_buffer_get_csrc (&rtp, 0)); fail_unless_equals_int (data[0] & 0xf, 0); gst_rtp_buffer_unmap (&rtp); @@ -128,7 +130,9 @@ GST_START_TEST (test_rtp_buffer) gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp); fail_unless_equals_int (gst_rtp_buffer_get_csrc_count (&rtp), 3); - ASSERT_CRITICAL (gst_rtp_buffer_get_csrc (&rtp, 3)); + GST_FIXME + ("Unable to check CSRC. ASSERT_CRITICAL is broken. See MCU issue 21304"); + // ASSERT_CRITICAL (gst_rtp_buffer_get_csrc (&rtp, 3)); fail_unless_equals_int (data[0] & 0xf, 3); fail_unless_equals_int (gst_rtp_buffer_get_csrc (&rtp, 0), 0); fail_unless_equals_int (gst_rtp_buffer_get_csrc (&rtp, 1), 0); @@ -144,7 +148,9 @@ GST_START_TEST (test_rtp_buffer) fail_unless_equals_int (GST_READ_UINT32_BE (data + 1 * 4), 0xf7c1); gst_rtp_buffer_set_csrc (&rtp, 2, 0xf7c2); fail_unless_equals_int (GST_READ_UINT32_BE (data + 2 * 4), 0xf7c2); - ASSERT_CRITICAL (gst_rtp_buffer_set_csrc (&rtp, 3, 0xf123)); + GST_FIXME + ("Unable to check CSRC. ASSERT_CRITICAL is broken. See MCU issue 21304"); + // ASSERT_CRITICAL (gst_rtp_buffer_set_csrc (&rtp, 3, 0xf123)); gst_rtp_buffer_unmap (&rtp); gst_buffer_unmap (buf, &map); From 390f16282b9377ed6736a8fae5a5f05630e311c5 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 7 Apr 2020 10:34:16 +0200 Subject: [PATCH 124/339] rtpbuffer: conversions between video ROI meta and RTP extension header A way to embed ROI meta in the RTP extension header. --- .../gst-libs/gst/rtp/gstrtpbuffer.c | 86 +++++++++++++++ .../gst-libs/gst/rtp/gstrtpbuffer.h | 10 ++ .../gst-libs/gst/rtp/meson.build | 2 +- .../gst-plugins-base/tests/check/libs/rtp.c | 103 ++++++++++++++++++ 4 files changed, 200 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.c index ec4b53bd7be..466558dc0b5 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.c @@ -34,6 +34,7 @@ #endif #include "gstrtpbuffer.h" +#include #include #include @@ -1766,3 +1767,88 @@ gst_rtp_buffer_add_extension_twobytes_header (GstRTPBuffer * rtp, return TRUE; } + +/** + * gst_rtp_buffer_video_roi_meta_to_one_byte_ext: + * @rtp: a #GstRTPBuffer to write a ROI header extension to. + * @buffer: a #GstBuffer to extract #GstVideoRegionOfInterestMeta from. + * @id: The ID of the header extension to be added (between 1 and 14). + * + * Tries to read #GstVideoRegionOfInterestMeta from to the @buffer, and + * add this as ROI information in @rtp using the one byte + * extension header. + * + * Returns: %TRUE if any extension-headers was added, %FALSE otherwise. + * + * Since: 1.18 + */ +gboolean +gst_rtp_buffer_video_roi_meta_to_one_byte_ext (GstRTPBuffer * rtp, + GstBuffer * buffer, guint8 id) +{ + GstVideoRegionOfInterestMeta *meta; + gpointer state = NULL; + gboolean ret = FALSE; + + g_return_val_if_fail (buffer != NULL, FALSE); + g_return_val_if_fail (rtp != NULL, FALSE); + g_return_val_if_fail (id > 0, FALSE); + g_return_val_if_fail (id <= 14, FALSE); + + while ((meta = (GstVideoRegionOfInterestMeta *) + gst_buffer_iterate_meta_filtered (buffer, &state, + GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE))) { + guint8 data[9]; + GST_WRITE_UINT16_BE (&data[0], meta->x); + GST_WRITE_UINT16_BE (&data[2], meta->y); + GST_WRITE_UINT16_BE (&data[4], meta->w); + GST_WRITE_UINT16_BE (&data[6], meta->h); + GST_WRITE_UINT8 (&data[8], meta->roi_type); + gst_rtp_buffer_add_extension_onebyte_header (rtp, id, &data, 9); + ret = TRUE; + } + + return ret; +} + +/** + * gst_rtp_buffer_video_roi_meta_from_one_byte_ext: + * @rtp: a #GstRTPBuffer to read a ROI header extension to. + * @buffer: a #GstBuffer to write #GstVideoRegionOfInterestMeta to. + * @id: The ID of the header extension to be read (between 1 and 14). + * + * Tries to extract ROI information from the RTP packet + * and adds this as #GstVideoRegionOfInterestMeta on to the @buffer. + * + * Returns: %TRUE if any meta was extracted, %FALSE otherwise. + * + * Since: 1.18 + */ +gboolean +gst_rtp_buffer_video_roi_meta_from_one_byte_ext (GstRTPBuffer * rtp, + GstBuffer * buffer, guint8 id) +{ + gpointer data; + guint nth = 0; + + g_return_val_if_fail (buffer != NULL, FALSE); + g_return_val_if_fail (rtp != NULL, FALSE); + g_return_val_if_fail (id > 0, FALSE); + g_return_val_if_fail (id <= 14, FALSE); + + while (gst_rtp_buffer_get_extension_onebyte_header (rtp, id, nth, &data, + NULL)) { + const guint8 *bytes = (const guint8 *) data; + guint16 x = GST_READ_UINT16_BE (&bytes[0]); + guint16 y = GST_READ_UINT16_BE (&bytes[2]); + guint16 w = GST_READ_UINT16_BE (&bytes[4]); + guint16 h = GST_READ_UINT16_BE (&bytes[6]); + guint16 roi_type = GST_READ_UINT8 (&bytes[8]); + + gst_buffer_add_video_region_of_interest_meta_id (buffer, roi_type, x, y, w, + h); + nth++; + } + + return nth > 0; +} diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h index cac8998c2b8..a15c39b0f77 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h @@ -239,6 +239,16 @@ gboolean gst_rtp_buffer_get_extension_onebyte_header_from_bytes (GBytes * bytes, gpointer * data, guint * size); +GST_RTP_API +gboolean gst_rtp_buffer_video_roi_meta_to_one_byte_ext (GstRTPBuffer * rtp, + GstBuffer * buf, + guint8 id); + +GST_RTP_API +gboolean gst_rtp_buffer_video_roi_meta_from_one_byte_ext (GstRTPBuffer * rtp, + GstBuffer * buffer, + guint8 id); + /** * GstRTPBufferFlags: * @GST_RTP_BUFFER_FLAG_RETRANSMISSION: The #GstBuffer was once wrapped diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build b/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build index 19a65ee7137..14bf1fdf1b8 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build @@ -34,7 +34,7 @@ rtp_enums = gnome.mkenums_simple('gstrtp-enumtypes', gstrtp_enum_c = rtp_enums[0] gstrtp_enum_h = rtp_enums[1] -gstrtp_deps = [audio_dep, gst_base_dep] +gstrtp_deps = [audio_dep, video_dep, gst_base_dep] gst_rtp = library('gstrtp-@0@'.format(api_version), rtp_sources, gstrtp_enum_c, gstrtp_enum_h, c_args : gst_plugins_base_args + ['-DBUILDING_GST_RTP', '-DG_LOG_DOMAIN="GStreamer-RTP"'], diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtp.c b/subprojects/gst-plugins-base/tests/check/libs/rtp.c index 4771f351b38..a4aa037e4dd 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtp.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtp.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #define RTP_HEADER_LEN 12 @@ -2329,6 +2330,106 @@ GST_START_TEST (test_rtp_buffer_remove_extension_data) GST_END_TEST; +typedef struct +{ + const gchar *name; + guint x; + guint y; + guint w; + guint h; +} ROI; + +static void +_add_roi (GstBuffer * buf, ROI * roi) +{ + gst_buffer_add_video_region_of_interest_meta (buf, + roi->name, roi->x, roi->y, roi->w, roi->h); +} + +static void +_meta_to_roi (GstVideoRegionOfInterestMeta * meta, ROI * roi) +{ + roi->x = meta->x; + roi->y = meta->y; + roi->w = meta->w; + roi->h = meta->h; +} + +static gboolean +_foreach_meta_func (G_GNUC_UNUSED GstBuffer * buf, GstMeta ** meta, + gpointer user_data) +{ + GArray *array = user_data; + const GstMetaInfo *info = (*meta)->info; + + if (info->api == GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE) { + ROI roi; + _meta_to_roi ((GstVideoRegionOfInterestMeta *) * meta, &roi); + g_array_append_val (array, roi); + } + + return TRUE; +} + +static GArray * +_get_rois (GstBuffer * buf) +{ + GArray *array = g_array_new (FALSE, FALSE, sizeof (ROI)); + gst_buffer_foreach_meta (buf, _foreach_meta_func, array); + return array; +} + +GST_START_TEST (test_rtp_buffer_roi_meta_as_ext_hdr) +{ + GstBuffer *buf0 = gst_buffer_new (); + GstBuffer *buf1 = gst_buffer_new (); + GstBuffer *rtpbuf = gst_rtp_buffer_new_allocate (10, 0, 0); + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GArray *result_rois; + guint i; + ROI expected_rois[] = { + {"test0", 10, 10, 10, 10} + , + {"test1", 20, 20, 20, 20} + , + {"test2", 30, 30, 30, 30} + , + }; + + /* add ROI meta to buf0 */ + for (i = 0; i < G_N_ELEMENTS (expected_rois); i++) + _add_roi (buf0, &expected_rois[i]); + + /* ROI meta -> RTP header extension */ + gst_rtp_buffer_map (rtpbuf, GST_MAP_READWRITE, &rtp); + gst_rtp_buffer_video_roi_meta_to_one_byte_ext (&rtp, buf0, 10); + gst_rtp_buffer_unmap (&rtp); + + /* RTP header extension -> ROI meta */ + gst_rtp_buffer_map (rtpbuf, GST_MAP_READWRITE, &rtp); + gst_rtp_buffer_video_roi_meta_from_one_byte_ext (&rtp, buf1, 10); + gst_rtp_buffer_unmap (&rtp); + + /* verify the meta survived the conversion */ + result_rois = _get_rois (buf1); + fail_unless_equals_int (result_rois->len, G_N_ELEMENTS (expected_rois)); + for (i = 0; i < G_N_ELEMENTS (expected_rois); i++) { + ROI *exp = &expected_rois[i]; + ROI *act = &g_array_index (result_rois, ROI, i); + fail_unless_equals_int (exp->x, act->x); + fail_unless_equals_int (exp->y, act->y); + fail_unless_equals_int (exp->w, act->w); + fail_unless_equals_int (exp->h, act->h); + } + g_array_unref (result_rois); + + gst_buffer_unref (rtpbuf); + gst_buffer_unref (buf0); + gst_buffer_unref (buf1); +} + +GST_END_TEST; + static Suite * rtp_suite (void) { @@ -2386,6 +2487,8 @@ rtp_suite (void) tcase_add_test (tc_chain, test_rtp_buffer_remove_extension_data); tcase_add_test (tc_chain, test_rtp_buffer_set_extension_data_shrink_data); + tcase_add_test (tc_chain, test_rtp_buffer_roi_meta_as_ext_hdr); + return s; } From d90bd29afeaee8b3f12854350ae193958c79dcb7 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 13 May 2020 16:06:46 +0200 Subject: [PATCH 125/339] rtpbuffer: add buffer-flags to mark media-type audio and video --- subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h index a15c39b0f77..b67add9a1c6 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h @@ -270,6 +270,8 @@ gboolean gst_rtp_buffer_video_roi_meta_from_one_byte_ext (GstRTPBuffer * rtp, typedef enum { GST_RTP_BUFFER_FLAG_RETRANSMISSION = (GST_BUFFER_FLAG_LAST << 0), GST_RTP_BUFFER_FLAG_REDUNDANT = (GST_BUFFER_FLAG_LAST << 1), + GST_RTP_BUFFER_FLAG_MEDIA_AUDIO = (GST_BUFFER_FLAG_LAST << 2), + GST_RTP_BUFFER_FLAG_MEDIA_VIDEO = (GST_BUFFER_FLAG_LAST << 3), GST_RTP_BUFFER_FLAG_LAST = (GST_BUFFER_FLAG_LAST << 8) } GstRTPBufferFlags; From 9fe8ceca7118edcb37116b46e39792eecc854197 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 18 Mar 2021 22:27:43 +0100 Subject: [PATCH 126/339] rtcpbuffer: allow Padding bit for non-compound packets ...and deal with non-modulo 4 padding bytes for non-standard RTCP. This is thanks to Chrome sending these packets to us when doing TWCC. Whereas the standard says that non-modulo 4 bytes padding are not allowed, for reduced size and TWCC in particular, this does not seem to apply. In order to be able to safely parse FCIs from these packets, a new API is proposed: gst_rtcp_packet_fb_get_fci_length_bytes, that will respect the non-mod4 padding and subtract that from the valid FCI length. Co-Authored-By: Tulio Beloqui Co-Authored-By: John-Mark Bell --- .../gst-libs/gst/rtp/gstrtcpbuffer.c | 73 +++++++--- .../gst-libs/gst/rtp/gstrtcpbuffer.h | 7 +- .../gst-plugins-base/tests/check/libs/rtp.c | 126 +++++++++++++++++- 3 files changed, 180 insertions(+), 26 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.c index dea2a7933d5..49664b679c5 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.c @@ -91,12 +91,9 @@ static gboolean gst_rtcp_buffer_validate_data_internal (guint8 * data, guint len, guint16 valid_mask) { - guint16 header_mask; - guint header_len; guint8 version; + guint16 header_mask; guint data_len; - gboolean padding; - guint8 pad_bytes; g_return_val_if_fail (data != NULL, FALSE); @@ -109,43 +106,49 @@ gst_rtcp_buffer_validate_data_internal (guint8 * data, guint len, if (G_UNLIKELY (header_mask != GST_RTCP_VALID_VALUE)) goto wrong_mask; - padding = data[0] & 0x20; - /* store len */ data_len = len; while (TRUE) { + guint header_len; + gboolean padding = FALSE; + /* get packet length */ header_len = (((data[2] << 8) | data[3]) + 1) << 2; if (data_len < header_len) goto wrong_length; + /* check padding of new packet */ + if (data[0] & 0x20) { + guint8 pad_bytes; + padding = TRUE; + /* last byte of padding contains the number of padded bytes including + * itself */ + pad_bytes = data[data_len - 1]; + /* cannot be 0 and must be smaller than the size of the data */ + if (pad_bytes == 0 || pad_bytes > (header_len - 4)) + goto wrong_padding; + /* if validating a standard RTCP packet, padding must be modulo 4 */ + if (GST_RTCP_VALID_MASK == valid_mask && (pad_bytes & 0x3)) + goto wrong_padding; + } + /* move to next compound packet */ data += header_len; data_len -= header_len; - /* we are at the end now */ - if (data_len < 4) + /* padding only allowed on last packet */ + if (padding) break; - /* Version already checked for first packet through mask */ - if (padding) + /* we are at the end now */ + if (data_len < 4) break; /* check version of new packet */ version = data[0] & 0xc0; if (version != (GST_RTCP_VERSION << 6)) goto wrong_version; - - /* check padding of new packet */ - if (data[0] & 0x20) { - padding = TRUE; - /* last byte of padding contains the number of padded bytes including - * itself. must be a multiple of 4, but cannot be 0. */ - pad_bytes = data[data_len - 1]; - if (pad_bytes == 0 || (pad_bytes & 0x3)) - goto wrong_padding; - } } if (data_len != 0) { /* some leftover bytes */ @@ -2324,6 +2327,36 @@ gst_rtcp_packet_fb_get_fci_length (GstRTCPPacket * packet) return GST_READ_UINT16_BE (data) - 2; } +/** + * gst_rtcp_packet_fb_get_fci_length_bytes: + * @packet: a valid RTPFB or PSFB #GstRTCPPacket + * + * Get the length of the Feedback Control Information attached to a + * RTPFB or PSFB @packet. + * + * Returns: The length of the FCI in bytes. + */ +guint16 +gst_rtcp_packet_fb_get_fci_length_bytes (GstRTCPPacket * packet) +{ + guint16 wordlen; + guint16 byteslen; + + wordlen = gst_rtcp_packet_fb_get_fci_length (packet); + byteslen = wordlen * 4; + + /* subtract any padding */ + if (packet->padding) { + guint8 *data; + guint8 pad_bytes; + data = packet->rtcp->map.data + packet->offset; + pad_bytes = data[4 + packet->length * 4 - 1]; + byteslen -= pad_bytes; + } + + return byteslen; +} + /** * gst_rtcp_packet_fb_set_fci_length: * @packet: a valid RTPFB or PSFB #GstRTCPPacket diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.h index b6410a5a1f5..4173d288b2c 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.h @@ -255,9 +255,9 @@ typedef enum /** * GST_RTCP_VALID_MASK: * - * Mask for version, padding bit and packet type pair + * Mask for version and packet type pair */ -#define GST_RTCP_VALID_MASK (0xc000 | 0x2000 | 0xfe) +#define GST_RTCP_VALID_MASK (0xc000 | 0xfe) /** * GST_RTCP_REDUCED_SIZE_VALID_MASK: @@ -546,6 +546,9 @@ void gst_rtcp_packet_fb_set_type (GstRTCPPacket *packet, Gs GST_RTP_API guint16 gst_rtcp_packet_fb_get_fci_length (GstRTCPPacket *packet); +GST_RTP_API +guint16 gst_rtcp_packet_fb_get_fci_length_bytes (GstRTCPPacket * packet); + GST_RTP_API gboolean gst_rtcp_packet_fb_set_fci_length (GstRTCPPacket *packet, guint16 wordlen); diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtp.c b/subprojects/gst-plugins-base/tests/check/libs/rtp.c index a4aa037e4dd..38b01ba237b 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtp.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtp.c @@ -941,7 +941,7 @@ GST_START_TEST (test_rtcp_validate_with_padding) /* Compound packet with padding in the last packet. Padding is included in * the length of the last packet. */ guint8 rtcp_pkt[] = { - 0x80, 0xC9, 0x00, 0x07, /* Type RR, length = 7 */ + 0x80, 0xc9, 0x00, 0x07, /* Type RR, length = 7 */ 0x97, 0x6d, 0x21, 0x6a, 0x4d, 0x16, 0xaf, 0x14, 0x10, 0x1f, 0xd9, 0x91, @@ -972,7 +972,7 @@ GST_START_TEST (test_rtcp_validate_with_padding_wrong_padlength) /* Compound packet with padding in the last packet. Padding is included in * the length of the last packet. */ guint8 rtcp_pkt[] = { - 0x80, 0xC9, 0x00, 0x07, /* Type RR, length = 7 */ + 0x80, 0xc9, 0x00, 0x07, /* Type RR, length = 7 */ 0x97, 0x6d, 0x21, 0x6a, 0x4d, 0x16, 0xaf, 0x14, 0x10, 0x1f, 0xd9, 0x91, @@ -1003,7 +1003,7 @@ GST_START_TEST (test_rtcp_validate_with_padding_excluded_from_length) /* Compound packet with padding in the last packet. Padding is not included * in the length. */ guint8 rtcp_pkt[] = { - 0x80, 0xC9, 0x00, 0x07, /* Type RR, length = 7 */ + 0x80, 0xc9, 0x00, 0x07, /* Type RR, length = 7 */ 0x97, 0x6d, 0x21, 0x6a, 0x4d, 0x16, 0xaf, 0x14, 0x10, 0x1f, 0xd9, 0x91, @@ -1034,7 +1034,7 @@ GST_START_TEST (test_rtcp_validate_with_padding_set_in_first_packet) /* Compound packet with padding in the last packet but with the pad bit set on first packet */ guint8 rtcp_pkt[] = { - 0xA0, 0xC9, 0x00, 0x07, /* P=1, Type RR, length = 7 */ + 0xa0, 0xc9, 0x00, 0x07, /* P=1, Type RR, length = 7 */ 0x97, 0x6d, 0x21, 0x6a, 0x4d, 0x16, 0xaf, 0x14, 0x10, 0x1f, 0xd9, 0x91, @@ -1099,6 +1099,118 @@ GST_START_TEST (test_rtcp_validate_reduced_with_padding) GST_END_TEST; +GST_START_TEST (test_rtcp_validate_reduced_with_invalid_padding) +{ + /* Reduced size packet with padding. */ + guint8 rtcp_pkt[] = { + 0xa0, 0xcd, 0x00, 0x01, /* P=1, PT=205(FB), length=1 (x4 = 4 bytes) */ + 0x00, 0x00, 0x00, 0x05 /* 5x padding bytes, more than total */ + }; + + fail_if (gst_rtcp_buffer_validate_data_reduced (rtcp_pkt, sizeof (rtcp_pkt))); +} + +GST_END_TEST; + +GST_START_TEST (test_rtcp_validate_reduced_with_invalid_padding_and_length) +{ + /* Reduced size packet with padding. */ + guint8 rtcp_pkt0[] = { + 0xa0, 0xcd, 0x00, 0x00, /* P=1, PT=205(FB), length=0 */ + }; + guint8 rtcp_pkt1[] = { + 0xa0, 0xcd, 0x00, 0x01, /* P=1, PT=205(FB), length=1 */ + }; + + fail_if (gst_rtcp_buffer_validate_data_reduced (rtcp_pkt0, + sizeof (rtcp_pkt0))); + fail_if (gst_rtcp_buffer_validate_data_reduced (rtcp_pkt1, + sizeof (rtcp_pkt1))); +} + +GST_END_TEST; + +GST_START_TEST (test_rtcp_validate_and_parse_chrome_twcc) +{ + GstBuffer *buffer; + GstRTCPPacket packet; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + + guint8 rtcp_pkt[] = { + 0x8f, 0xcd, 0x00, 0x05, /* P=0, FMT=15, PT=205(FB), length=5 (x4 = 20 bytes) */ + 0xce, 0x0d, 0xc2, 0xb3, /* SSRC of packet sender */ + 0x53, 0xf6, 0x11, 0x3b, /* SSRC of media source */ + 0x00, 0x37, 0x00, 0x02, /* start of FCI */ + 0x00, 0xce, 0x40, 0x02, + 0x20, 0x02, 0x84, 0x50 /* no padding */ + }; + buffer = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, + rtcp_pkt, sizeof (rtcp_pkt), 0, sizeof (rtcp_pkt), NULL, NULL); + + fail_unless (gst_rtcp_buffer_validate_reduced (buffer)); + + gst_rtcp_buffer_map (buffer, GST_MAP_READ, &rtcp); + + fail_unless (gst_rtcp_buffer_get_first_packet (&rtcp, &packet)); + + fail_unless_equals_int (GST_RTCP_TYPE_RTPFB, + gst_rtcp_packet_get_type (&packet)); + fail_unless_equals_int (GST_RTCP_RTPFB_TYPE_TWCC, + gst_rtcp_packet_fb_get_type (&packet)); + + fail_unless_equals_int (3, gst_rtcp_packet_fb_get_fci_length (&packet)); + fail_unless_equals_int (12, + gst_rtcp_packet_fb_get_fci_length_bytes (&packet)); + + gst_rtcp_buffer_unmap (&rtcp); + gst_buffer_unref (buffer); + +} + +GST_END_TEST; + +GST_START_TEST (test_rtcp_validate_and_parse_padded_chrome_twcc) +{ + GstBuffer *buffer; + GstRTCPPacket packet; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + + guint8 rtcp_pkt[] = { + 0xaf, 0xcd, 0x00, 0x06, /* P=1, FMT=15, FTPT=205(FB), length=6 (x4 = 24 bytes) */ + 0xce, 0x0d, 0xc2, 0xb3, /* SSRC of packet sender */ + 0x53, 0xf6, 0x11, 0x3b, /* SSRC of media source */ + 0x00, 0x34, 0x00, 0x03, /* start of FCI */ + 0x00, 0xce, 0x3f, 0x01, + 0x20, 0x03, 0x94, 0x50, + 0x50, 0x00, 0x00, 0x03 /* 3x padding bytes */ + }; + + buffer = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, + rtcp_pkt, sizeof (rtcp_pkt), 0, sizeof (rtcp_pkt), NULL, NULL); + + fail_unless (gst_rtcp_buffer_validate_reduced (buffer)); + + gst_rtcp_buffer_map (buffer, GST_MAP_READ, &rtcp); + + fail_unless (gst_rtcp_buffer_get_first_packet (&rtcp, &packet)); + + fail_unless_equals_int (GST_RTCP_TYPE_RTPFB, + gst_rtcp_packet_get_type (&packet)); + fail_unless_equals_int (GST_RTCP_RTPFB_TYPE_TWCC, + gst_rtcp_packet_fb_get_type (&packet)); + + /* This is actually "wrong", but the resolution for fci_length is in 32bits */ + fail_unless_equals_int (4, gst_rtcp_packet_fb_get_fci_length (&packet)); + /* hence the introduction of this API: */ + fail_unless_equals_int (13, + gst_rtcp_packet_fb_get_fci_length_bytes (&packet)); + + gst_rtcp_buffer_unmap (&rtcp); + gst_buffer_unref (buffer); +} + +GST_END_TEST; + GST_START_TEST (test_rtcp_buffer_profile_specific_extension) { GstBuffer *buf; @@ -2455,6 +2567,12 @@ rtp_suite (void) test_rtcp_validate_with_padding_set_in_first_packet); tcase_add_test (tc_chain, test_rtcp_validate_reduced_without_padding); tcase_add_test (tc_chain, test_rtcp_validate_reduced_with_padding); + tcase_add_test (tc_chain, test_rtcp_validate_reduced_with_invalid_padding); + tcase_add_test (tc_chain, + test_rtcp_validate_reduced_with_invalid_padding_and_length); + tcase_add_test (tc_chain, test_rtcp_validate_and_parse_chrome_twcc); + tcase_add_test (tc_chain, test_rtcp_validate_and_parse_padded_chrome_twcc); + tcase_add_test (tc_chain, test_rtcp_buffer_profile_specific_extension); tcase_add_test (tc_chain, test_rtcp_buffer_app); tcase_add_test (tc_chain, test_rtcp_buffer_xr); From bc6f6f1b8b3fd8bd2b377a007bcbf9959eb943bc Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 21 Apr 2021 17:16:07 +0200 Subject: [PATCH 127/339] videodecoder: added detect-reordering property. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't always want the PTS of the frame to be updated. The default value is kept TRUE to comply with the previous behaviour. Co-Authored-By: Håvard Graff --- .../gst-libs/gst/video/gstvideodecoder.c | 45 +++++++++--- .../tests/check/libs/videodecoder.c | 72 +++++++++++++++++++ 2 files changed, 109 insertions(+), 8 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c index 74bcab1e728..65c840f2912 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c @@ -314,6 +314,7 @@ GST_DEBUG_CATEGORY (videodecoder_debug); * frame numbers and can be given special meaning */ #define REQUEST_SYNC_POINT_PENDING G_MAXUINT + 1 #define REQUEST_SYNC_POINT_UNSET G_MAXUINT64 +#define DEFAULT_DETECT_REORDERING TRUE enum { @@ -324,6 +325,8 @@ enum PROP_DISCARD_CORRUPTED_FRAMES, PROP_AUTOMATIC_REQUEST_SYNC_POINTS, PROP_AUTOMATIC_REQUEST_SYNC_POINT_FLAGS, + PROP_DETECT_REORDERING, + PROP_LAST }; struct _GstVideoDecoderPrivate @@ -386,6 +389,7 @@ struct _GstVideoDecoderPrivate /* incoming pts - dts */ GstClockTime pts_delta; gboolean reordered_output; + gboolean detect_reordering; /* FIXME: Consider using a GQueue or other better fitting data structure */ /* reverse playback */ @@ -716,6 +720,21 @@ gst_video_decoder_class_init (GstVideoDecoderClass * klass) DEFAULT_AUTOMATIC_REQUEST_SYNC_POINT_FLAGS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** + * GstVideoDecoder:detect-reordering: + * + * If set to %FALSE the decoder will not bother with reordered frames, and + * do any sort of PTS adjustment for them. + * + * Since: 1.20 + */ + g_object_class_install_property (gobject_class, PROP_DETECT_REORDERING, + g_param_spec_boolean ("detect-reordering", + "Detect frame reordering", + "Enables the frame pts reordering detection", + DEFAULT_DETECT_REORDERING, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + meta_tag_video_quark = g_quark_from_static_string (GST_META_TAG_VIDEO_STR); } @@ -770,6 +789,7 @@ gst_video_decoder_init (GstVideoDecoder * decoder, GstVideoDecoderClass * klass) /* properties */ decoder->priv->do_qos = DEFAULT_QOS; decoder->priv->max_errors = GST_VIDEO_DECODER_MAX_ERRORS; + decoder->priv->detect_reordering = DEFAULT_DETECT_REORDERING; decoder->priv->min_latency = 0; decoder->priv->max_latency = 0; @@ -995,6 +1015,9 @@ gst_video_decoder_get_property (GObject * object, guint property_id, case PROP_AUTOMATIC_REQUEST_SYNC_POINT_FLAGS: g_value_set_flags (value, priv->automatic_request_sync_point_flags); break; + case PROP_DETECT_REORDERING: + g_value_set_boolean (value, priv->detect_reordering); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -1027,6 +1050,9 @@ gst_video_decoder_set_property (GObject * object, guint property_id, case PROP_AUTOMATIC_REQUEST_SYNC_POINT_FLAGS: priv->automatic_request_sync_point_flags = g_value_get_flags (value); break; + case PROP_DETECT_REORDERING: + priv->detect_reordering = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -3128,14 +3154,17 @@ gst_video_decoder_prepare_finish_frame (GstVideoDecoder * } } - if (GST_CLOCK_TIME_IS_VALID (priv->last_timestamp_out) && - frame->pts < priv->last_timestamp_out) { - GST_WARNING_OBJECT (decoder, - "decreasing timestamp (%" GST_TIME_FORMAT " < %" GST_TIME_FORMAT ")", - GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (priv->last_timestamp_out)); - priv->reordered_output = TRUE; - /* make it a bit less weird downstream */ - frame->pts = priv->last_timestamp_out; + if (priv->detect_reordering + && GST_CLOCK_TIME_IS_VALID (priv->last_timestamp_out)) { + if (frame->pts < priv->last_timestamp_out) { + GST_WARNING_OBJECT (decoder, + "decreasing timestamp (%" GST_TIME_FORMAT " < %" + GST_TIME_FORMAT ")", + GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (priv->last_timestamp_out)); + priv->reordered_output = TRUE; + /* make it a bit less weird downstream */ + frame->pts = priv->last_timestamp_out; + } } if (GST_CLOCK_TIME_IS_VALID (frame->pts)) diff --git a/subprojects/gst-plugins-base/tests/check/libs/videodecoder.c b/subprojects/gst-plugins-base/tests/check/libs/videodecoder.c index c995321e20d..4008a0a41a6 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/videodecoder.c +++ b/subprojects/gst-plugins-base/tests/check/libs/videodecoder.c @@ -1578,7 +1578,76 @@ GST_START_TEST (videodecoder_playback_invalid_ts_packetized_subframes) GST_END_TEST; +static void +test_videodecoder_property_detect_reordering (gboolean detect_reordering) +{ + GstSegment segment; + GstBuffer *buffer; + GstClockTime last_timestamp = GST_CLOCK_TIME_NONE; + guint i = 0; + + GstClockTime timestamps[] = { + 1000, + 2000, + 1500, + 2500, + 3000 + }; + + setup_videodecodertester (NULL, NULL); + + gst_pad_set_active (mysrcpad, TRUE); + g_object_set (dec, "detect-reordering", detect_reordering, NULL); + gst_element_set_state (dec, GST_STATE_PLAYING); + gst_pad_set_active (mysinkpad, TRUE); + + send_startup_events (); + + /* push a new segment */ + gst_segment_init (&segment, GST_FORMAT_TIME); + fail_unless (gst_pad_push_event (mysrcpad, gst_event_new_segment (&segment))); + + /* push a number of buffers with their corresponding timestamp */ + for (i = 0; i < G_N_ELEMENTS (timestamps); i++) { + buffer = create_test_buffer (i); + GST_BUFFER_PTS (buffer) = timestamps[i]; + fail_unless (gst_pad_push (mysrcpad, buffer) == GST_FLOW_OK); + } + for (i = 0; i < g_list_length (buffers); i++) { + buffer = g_list_nth_data (buffers, i); + + if (detect_reordering) { + /* when reordering we expect PTS to be continuous */ + fail_unless (GST_CLOCK_DIFF (last_timestamp, + GST_BUFFER_PTS (buffer)) >= 0); + last_timestamp = GST_BUFFER_PTS (buffer); + + } else { + /* we expect PTS not to be modified */ + fail_unless_equals_int (GST_BUFFER_PTS (buffer), timestamps[i]); + } + } + + g_list_free_full (buffers, (GDestroyNotify) gst_buffer_unref); + buffers = NULL; + + cleanup_videodecodertest (); +} + +GST_START_TEST (videodecoder_property_detect_reordering_enabled) +{ + test_videodecoder_property_detect_reordering (TRUE); +} + +GST_END_TEST; + +GST_START_TEST (videodecoder_property_detect_reordering_disabled) +{ + test_videodecoder_property_detect_reordering (FALSE); +} + +GST_END_TEST; static Suite * gst_videodecoder_suite (void) @@ -1616,6 +1685,9 @@ gst_videodecoder_suite (void) tcase_add_test (tc, videodecoder_playback_invalid_ts_packetized); tcase_add_test (tc, videodecoder_playback_invalid_ts_packetized_subframes); + tcase_add_test (tc, videodecoder_property_detect_reordering_enabled); + tcase_add_test (tc, videodecoder_property_detect_reordering_disabled); + return s; } From d08b81852256fe2c5404ed09bd033f0219d52e88 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 21 Apr 2021 17:45:46 +0200 Subject: [PATCH 128/339] audiodecoder: [pexhack] allow setting reported latency as a property --- .../gst-libs/gst/audio/gstaudiodecoder.c | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c index 8d9357af22e..ec61c85cc53 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c @@ -141,12 +141,14 @@ enum { PROP_0, PROP_LATENCY, + PROP_REPORTED_MIN_LATENCY, PROP_TOLERANCE, PROP_PLC, PROP_MAX_ERRORS }; #define DEFAULT_LATENCY 0 +#define DEFAULT_REPORTED_MIN_LATENCY 0 #define DEFAULT_TOLERANCE 0 #define DEFAULT_PLC FALSE #define DEFAULT_DRAINABLE TRUE @@ -264,6 +266,7 @@ struct _GstAudioDecoderPrivate /* properties */ GstClockTime latency; + GstClockTime reported_min_latency; GstClockTime tolerance; gboolean plc; gboolean drainable; @@ -406,6 +409,13 @@ gst_audio_decoder_class_init (GstAudioDecoderClass * klass) 0, G_MAXINT64, DEFAULT_LATENCY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_REPORTED_MIN_LATENCY, + g_param_spec_int64 ("reported-min-latency", + "The Reported Minimum Latency", + "The minimum latency reported to the latency query", 0, G_MAXINT64, + DEFAULT_REPORTED_MIN_LATENCY, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_TOLERANCE, g_param_spec_int64 ("tolerance", "Tolerance", "Perfect ts while timestamp jitter/imperfection within tolerance (ns)", @@ -496,6 +506,7 @@ gst_audio_decoder_init (GstAudioDecoder * dec, GstAudioDecoderClass * klass) /* property default */ dec->priv->latency = DEFAULT_LATENCY; + dec->priv->reported_min_latency = DEFAULT_REPORTED_MIN_LATENCY; dec->priv->tolerance = DEFAULT_TOLERANCE; dec->priv->plc = DEFAULT_PLC; dec->priv->drainable = DEFAULT_DRAINABLE; @@ -1068,6 +1079,11 @@ gst_audio_decoder_output (GstAudioDecoder * dec, GstBuffer * buf) GST_TIME_ARGS (GST_BUFFER_DURATION (buf))); } + if (priv->ctx.min_latency != priv->reported_min_latency) { + gst_audio_decoder_set_latency (dec, + priv->reported_min_latency, priv->reported_min_latency); + } + again: inbuf = NULL; if (priv->agg && dec->priv->latency > 0 && @@ -3137,6 +3153,9 @@ gst_audio_decoder_get_property (GObject * object, guint prop_id, case PROP_LATENCY: g_value_set_int64 (value, dec->priv->latency); break; + case PROP_REPORTED_MIN_LATENCY: + g_value_set_int64 (value, dec->priv->ctx.min_latency); + break; case PROP_TOLERANCE: g_value_set_int64 (value, dec->priv->tolerance); break; @@ -3164,6 +3183,9 @@ gst_audio_decoder_set_property (GObject * object, guint prop_id, case PROP_LATENCY: dec->priv->latency = g_value_get_int64 (value); break; + case PROP_REPORTED_MIN_LATENCY: + dec->priv->reported_min_latency = g_value_get_int64 (value); + break; case PROP_TOLERANCE: dec->priv->tolerance = g_value_get_int64 (value); break; @@ -3432,9 +3454,12 @@ gst_audio_decoder_set_latency (GstAudioDecoder * dec, GST_OBJECT_UNLOCK (dec); /* post latency message on the bus */ - if (post_message) + if (post_message) { gst_element_post_message (GST_ELEMENT (dec), gst_message_new_latency (GST_OBJECT (dec))); + + gst_audio_decoder_push_event (dec, gst_event_new_latency_changed ()); + } } /** From b80ee2804c8cc0b5a128e9a7756b3b9e5fb6837b Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 31 May 2021 11:21:24 +0200 Subject: [PATCH 129/339] gstrtpbuffer: added GST_RTP_BUFFER_FLAG_ULPFEC flag to mark ULPFEC packets --- subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h index b67add9a1c6..8b08b57d292 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h @@ -272,6 +272,7 @@ typedef enum { GST_RTP_BUFFER_FLAG_REDUNDANT = (GST_BUFFER_FLAG_LAST << 1), GST_RTP_BUFFER_FLAG_MEDIA_AUDIO = (GST_BUFFER_FLAG_LAST << 2), GST_RTP_BUFFER_FLAG_MEDIA_VIDEO = (GST_BUFFER_FLAG_LAST << 3), + GST_RTP_BUFFER_FLAG_ULPFEC = (GST_BUFFER_FLAG_LAST << 4), GST_RTP_BUFFER_FLAG_LAST = (GST_BUFFER_FLAG_LAST << 8) } GstRTPBufferFlags; From a2b757e17770d64b28295eed03236404af22d9e3 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 2 Sep 2021 10:58:03 +0200 Subject: [PATCH 130/339] test/rtp: warningfixes --- .../tests/check/libs/rtpbasedepayload.c | 8 ++++---- .../tests/check/libs/rtpbasepayload.c | 7 ++++--- .../tests/check/libs/rtpdummyhdrextimpl.c | 13 +++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c b/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c index 29e9098663e..d006acc6360 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c @@ -186,7 +186,6 @@ gst_rtp_dummy_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) break; case GST_RTP_DUMMY_USE_PUSH_LIST_FUNC:{ GstBufferList *blist = gst_buffer_list_new (); - gint i; gst_buffer_list_add (blist, outbuf); for (i = 0; i != self->num_buffers_in_blist - 1; ++i) { gst_buffer_list_add (blist, gst_buffer_copy (outbuf)); @@ -1679,7 +1678,7 @@ GST_START_TEST (rtp_base_depayload_two_byte_hdr_ext) GST_END_TEST; static GstRTPHeaderExtension * -request_extension (GstRTPBaseDepayload * depayload, guint ext_id, +request_extension (G_GNUC_UNUSED GstRTPBaseDepayload * depayload, guint ext_id, const gchar * ext_uri, gpointer user_data) { GstRTPHeaderExtension *ext = user_data; @@ -1815,8 +1814,9 @@ GST_START_TEST (rtp_base_depayload_multiple_exts) GST_END_TEST; static GstRTPHeaderExtension * -request_extension_ignored (GstRTPBaseDepayload * depayload, guint ext_id, - const gchar * ext_uri, gpointer user_data) +request_extension_ignored (G_GNUC_UNUSED GstRTPBaseDepayload * depayload, + G_GNUC_UNUSED guint ext_id, G_GNUC_UNUSED const gchar * ext_uri, + gpointer user_data) { guint *request_counter = user_data; diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpbasepayload.c b/subprojects/gst-plugins-base/tests/check/libs/rtpbasepayload.c index a7cfa46ffd2..9883f624285 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpbasepayload.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpbasepayload.c @@ -2146,7 +2146,7 @@ GST_STATIC_PAD_TEMPLATE ("sink", DUMMY_HDR_EXT_URI)); static GstRTPHeaderExtension * -request_extension (GstRTPBasePayload * depayload, guint ext_id, +request_extension (G_GNUC_UNUSED GstRTPBasePayload * payload, guint ext_id, const gchar * ext_uri, gpointer user_data) { GstRTPHeaderExtension *ext = user_data; @@ -2199,8 +2199,9 @@ GST_START_TEST (rtp_base_payload_caps_request) GST_END_TEST; static GstRTPHeaderExtension * -request_extension_ignored (GstRTPBasePayload * depayload, guint ext_id, - const gchar * ext_uri, gpointer user_data) +request_extension_ignored (G_GNUC_UNUSED GstRTPBasePayload * payload, + G_GNUC_UNUSED guint ext_id, G_GNUC_UNUSED const gchar * ext_uri, + gpointer user_data) { guint *request_counter = user_data; diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c b/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c index 9910d9d61d5..8eee22d2047 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c @@ -160,8 +160,8 @@ gst_rtp_dummy_hdr_ext_get_supported_flags (GstRTPHeaderExtension * ext) } static gsize -gst_rtp_dummy_hdr_ext_get_max_size (GstRTPHeaderExtension * ext, - const GstBuffer * input_meta) +gst_rtp_dummy_hdr_ext_get_max_size (G_GNUC_UNUSED GstRTPHeaderExtension * ext, + G_GNUC_UNUSED const GstBuffer * input_meta) { GstRTPDummyHdrExt *dummy = GST_RTP_DUMMY_HDR_EXT (ext); @@ -172,8 +172,9 @@ gst_rtp_dummy_hdr_ext_get_max_size (GstRTPHeaderExtension * ext, static gssize gst_rtp_dummy_hdr_ext_write (GstRTPHeaderExtension * ext, - const GstBuffer * input_meta, GstRTPHeaderExtensionFlags write_flags, - GstBuffer * output, guint8 * data, gsize size) + G_GNUC_UNUSED const GstBuffer * input_meta, + G_GNUC_UNUSED GstRTPHeaderExtensionFlags write_flags, + G_GNUC_UNUSED GstBuffer * output, guint8 * data, gsize size) { GstRTPDummyHdrExt *dummy = GST_RTP_DUMMY_HDR_EXT (ext); @@ -188,8 +189,8 @@ gst_rtp_dummy_hdr_ext_write (GstRTPHeaderExtension * ext, static gboolean gst_rtp_dummy_hdr_ext_read (GstRTPHeaderExtension * ext, - GstRTPHeaderExtensionFlags read_flags, const guint8 * data, - gsize size, GstBuffer * buffer) + G_GNUC_UNUSED GstRTPHeaderExtensionFlags read_flags, const guint8 * data, + G_GNUC_UNUSED gsize size, G_GNUC_UNUSED GstBuffer * buffer) { GstRTPDummyHdrExt *dummy = GST_RTP_DUMMY_HDR_EXT (ext); From 3e5e7e06cb5869793d87f64c0f68f1a80df37487 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 1 Feb 2021 16:52:46 +0100 Subject: [PATCH 131/339] rtpbasedpayload: add "ignore-gaps" property Basically, don't set a DISCONT flag when detecting a seqnum jump. This is useful for ULPFEC, as a FEC packet might be hiding inside the original stream, and this stream is actually completely valid, even if the FEC packet is removed. --- .../gst-libs/gst/rtp/gstrtpbasedepayload.c | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c index c788898ef28..104f4ed434f 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c @@ -91,6 +91,7 @@ struct _GstRTPBaseDepayloadPrivate guint32 last_seqnum; guint32 last_rtptime; guint32 next_seqnum; + gboolean ignore_gaps; gint max_reorder; gboolean auto_hdr_ext; @@ -128,6 +129,7 @@ enum static guint gst_rtp_base_depayload_signals[LAST_SIGNAL] = { 0 }; +#define DEFAULT_IGNORE_GAPS FALSE #define DEFAULT_SOURCE_INFO FALSE #define DEFAULT_MAX_REORDER 100 #define DEFAULT_AUTO_HEADER_EXTENSION TRUE @@ -137,6 +139,7 @@ enum PROP_0, PROP_STATS, PROP_SOURCE_INFO, + PROP_IGNORE_GAPS, PROP_MAX_REORDER, PROP_AUTO_HEADER_EXTENSION, PROP_LAST @@ -345,6 +348,19 @@ gst_rtp_base_depayload_class_init (GstRTPBaseDepayloadClass * klass) DEFAULT_AUTO_HEADER_EXTENSION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** + * GstRTPBaseDepayload:ignore-gaps: + * + * Wether the depayoader should looks at sequence-number gaps, + * and set a DISCONT flag accordingly. + * + * Since: 1.20 + **/ + g_object_class_install_property (gobject_class, PROP_IGNORE_GAPS, + g_param_spec_boolean ("ignore-gaps", "Ignore Gaps", + "Wether the depayloader should ignore sequencenumber gaps", + DEFAULT_IGNORE_GAPS, G_PARAM_READWRITE)); + /** * GstRTPBaseDepayload::request-extension: * @object: the #GstRTPBaseDepayload @@ -446,6 +462,7 @@ gst_rtp_base_depayload_init (GstRTPBaseDepayload * filter, priv->duration = -1; priv->ref_ts = -1; priv->source_info = DEFAULT_SOURCE_INFO; + priv->ignore_gaps = DEFAULT_IGNORE_GAPS; priv->max_reorder = DEFAULT_MAX_REORDER; priv->auto_hdr_ext = DEFAULT_AUTO_HEADER_EXTENSION; priv->hdrext_aggregate = FALSE; @@ -794,7 +811,7 @@ gst_rtp_base_depayload_handle_buffer (GstRTPBaseDepayload * filter, gap = gst_rtp_buffer_compare_seqnum (seqnum, priv->next_seqnum); /* if we have no gap, all is fine */ - if (G_UNLIKELY (gap != 0)) { + if (G_UNLIKELY (gap != 0) && !priv->ignore_gaps) { GST_LOG_OBJECT (filter, "got packet %u, expected %u, gap %d", seqnum, priv->next_seqnum, gap); if (gap < 0) { @@ -1757,6 +1774,9 @@ gst_rtp_base_depayload_set_property (GObject * object, guint prop_id, gst_rtp_base_depayload_set_source_info_enabled (depayload, g_value_get_boolean (value)); break; + case PROP_IGNORE_GAPS: + priv->ignore_gaps = g_value_get_boolean (value); + break; case PROP_MAX_REORDER: priv->max_reorder = g_value_get_int (value); break; @@ -1788,6 +1808,9 @@ gst_rtp_base_depayload_get_property (GObject * object, guint prop_id, g_value_set_boolean (value, gst_rtp_base_depayload_is_source_info_enabled (depayload)); break; + case PROP_IGNORE_GAPS: + g_value_set_boolean (value, priv->ignore_gaps); + break; case PROP_MAX_REORDER: g_value_set_int (value, priv->max_reorder); break; From 6b623a2d1570cf490ac35ea67263a2c012693ea3 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 10 Sep 2021 16:10:05 +0200 Subject: [PATCH 132/339] rtpbasedepayload: set the GAP_FLAG_MISSING_DATA when there is actual loss --- .../gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c index 104f4ed434f..d683f6d12de 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c @@ -1641,6 +1641,7 @@ gst_rtp_base_depayload_packet_lost (GstRTPBaseDepayload * filter, const GstStructure *s; gboolean might_have_been_fec; gboolean res = TRUE; + gboolean noloss = FALSE; s = gst_event_get_structure (event); @@ -1654,6 +1655,7 @@ gst_rtp_base_depayload_packet_lost (GstRTPBaseDepayload * filter, "Packet loss event without timestamp or duration"); return FALSE; } + gst_structure_get_boolean (s, "no-packet-loss", &noloss); sevent = gst_pad_get_sticky_event (filter->srcpad, GST_EVENT_SEGMENT, 0); if (G_UNLIKELY (!sevent)) { @@ -1668,7 +1670,8 @@ gst_rtp_base_depayload_packet_lost (GstRTPBaseDepayload * filter, &might_have_been_fec) || !might_have_been_fec) { /* send GAP event */ sevent = gst_event_new_gap (timestamp, duration); - gst_event_set_gap_flags (sevent, GST_GAP_FLAG_MISSING_DATA); + if (!noloss) + gst_event_set_gap_flags (sevent, GST_GAP_FLAG_MISSING_DATA); res = gst_pad_push_event (filter->srcpad, sevent); } From d76a6d9c7aa7e90ede0917fc7f00052a50d67575 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 27 Sep 2021 02:05:21 +0200 Subject: [PATCH 133/339] videodecoder: only convert to running time if the input_segment is valid --- .../gst-libs/gst/video/gstvideodecoder.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c index 65c840f2912..eb5d2f6473a 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c @@ -1342,9 +1342,12 @@ gst_video_decoder_handle_missing_data_default (GstVideoDecoder * priv = decoder->priv; if (priv->automatic_request_sync_points) { - GstClockTime deadline = - gst_segment_to_running_time (&decoder->input_segment, GST_FORMAT_TIME, - timestamp); + GstClockTime deadline = timestamp; + + if (decoder->input_segment.format == GST_FORMAT_TIME) + deadline = + gst_segment_to_running_time (&decoder->input_segment, GST_FORMAT_TIME, + timestamp); GST_DEBUG_OBJECT (decoder, "Requesting sync point for missing data at running time %" From 48bc94b4be83e0deef8c3b21d4a2b791a232fa8f Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 28 Aug 2015 12:10:20 +0200 Subject: [PATCH 134/339] speexenc: Don't set lookahead Also introducing a test-suite for speex. https://bugzilla.gnome.org/show_bug.cgi?id=754226 --- .../gst-plugins-good/ext/speex/gstspeexenc.c | 5 +- tests/check/elements/speex.c | 107 ++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 tests/check/elements/speex.c diff --git a/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c b/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c index 3eac649a398..c834a10b69b 100644 --- a/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c +++ b/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c @@ -314,7 +314,10 @@ gst_speex_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) /* feedback to base class */ gst_audio_encoder_set_latency (benc, gst_speex_enc_get_latency (enc), gst_speex_enc_get_latency (enc)); - gst_audio_encoder_set_lookahead (benc, enc->lookahead); + + /* FIXME: awaiting a better solution, as lookahead does too much + wrong with both timestamp and duration. */ + //gst_audio_encoder_set_lookahead (benc, enc->lookahead); if (enc->nframes == 0) { /* as many frames as available input allows */ diff --git a/tests/check/elements/speex.c b/tests/check/elements/speex.c new file mode 100644 index 00000000000..6cc3bb671ea --- /dev/null +++ b/tests/check/elements/speex.c @@ -0,0 +1,107 @@ +/* GStreamer + * + * unit test for speex + * + * Copyright (C) <2017> Havard Graff + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +GST_START_TEST (test_encoder_timestamp) +{ + GstHarness * h = gst_harness_new ("speexenc"); + GstBuffer * buf; + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\"", TRUE); + + /* push 20ms of audio-data */ + gst_harness_push_from_src (h); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* push another 20ms of audio-data */ + gst_harness_push_from_src (h); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_encoder_to_decoder_timestamp) +{ + GstHarness * h = gst_harness_new_parse ("speexenc ! speexdec"); + GstBuffer * buf; + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\"", TRUE); + + /* push 20ms of audio-data */ + gst_harness_push_from_src (h); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* push another 20ms of audio-data */ + gst_harness_push_from_src (h); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} +GST_END_TEST; + +static Suite * +speex_suite (void) +{ + Suite *s = suite_create ("speex"); + TCase *tc_chain = tcase_create ("speex"); + + suite_add_tcase (s, tc_chain); + tcase_add_test (tc_chain, test_encoder_timestamp); + tcase_add_test (tc_chain, test_encoder_to_decoder_timestamp); + + return s; +} + +GST_CHECK_MAIN (speex) From 2d876e6ad25dad4562942b8d1b8d08fd44c642b9 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 23 Sep 2015 16:05:55 +0200 Subject: [PATCH 135/339] speexdec: Handle streamheaders in caps/buffers better https://bugzilla.gnome.org/show_bug.cgi?id=755481 --- .../gst-plugins-good/ext/speex/gstspeexdec.c | 99 +++++++++------ tests/check/elements/speex.c | 113 +++++++++++++++++- 2 files changed, 172 insertions(+), 40 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/speex/gstspeexdec.c b/subprojects/gst-plugins-good/ext/speex/gstspeexdec.c index eeb7af7e8b3..6d1d2d103ba 100644 --- a/subprojects/gst-plugins-good/ext/speex/gstspeexdec.c +++ b/subprojects/gst-plugins-good/ext/speex/gstspeexdec.c @@ -51,6 +51,8 @@ GST_DEBUG_CATEGORY_STATIC (speexdec_debug); #define DEFAULT_ENH TRUE +#define GST_FLOW_NO_HEADER GST_FLOW_CUSTOM_ERROR + enum { ARG_0, @@ -195,6 +197,7 @@ gst_speex_dec_stop (GstAudioDecoder * dec) static GstFlowReturn gst_speex_dec_parse_header (GstSpeexDec * dec, GstBuffer * buf) { + SpeexHeader *header; GstMapInfo map; GstAudioInfo info; static const GstAudioChannelPosition chan_pos[2][2] = { @@ -205,12 +208,18 @@ gst_speex_dec_parse_header (GstSpeexDec * dec, GstBuffer * buf) /* get the header */ gst_buffer_map (buf, &map, GST_MAP_READ); - dec->header = speex_packet_to_header ((gchar *) map.data, map.size); + header = speex_packet_to_header ((gchar *) map.data, map.size); gst_buffer_unmap (buf, &map); - if (!dec->header) + if (!header) goto no_header; + if (dec->header) { + GST_DEBUG_OBJECT (dec, "Replacing speex-header, resetting state"); + gst_speex_dec_reset (dec); + } + dec->header = header; + if (dec->header->mode >= SPEEX_NB_MODES || dec->header->mode < 0) goto mode_too_old; @@ -254,9 +263,8 @@ gst_speex_dec_parse_header (GstSpeexDec * dec, GstBuffer * buf) /* ERRORS */ no_header: { - GST_ELEMENT_ERROR (GST_ELEMENT (dec), STREAM, DECODE, - (NULL), ("couldn't read header")); - return GST_FLOW_ERROR; + GST_INFO_OBJECT (dec, "couldn't read header"); + return GST_FLOW_NO_HEADER; } mode_too_old: { @@ -290,7 +298,7 @@ gst_speex_dec_parse_comments (GstSpeexDec * dec, GstBuffer * buf) if (!list) { GST_WARNING_OBJECT (dec, "couldn't decode comments"); - list = gst_tag_list_new_empty (); + return GST_FLOW_NO_HEADER; } if (encoder) { @@ -473,6 +481,9 @@ memcmp_buffers (GstBuffer * buf1, GstBuffer * buf2) gsize size1, size2; gboolean res; + if (buf1 == NULL || buf2 == NULL) + return FALSE; + size1 = gst_buffer_get_size (buf1); size2 = gst_buffer_get_size (buf2); @@ -489,8 +500,9 @@ memcmp_buffers (GstBuffer * buf1, GstBuffer * buf2) static GstFlowReturn gst_speex_dec_handle_frame (GstAudioDecoder * bdec, GstBuffer * buf) { - GstFlowReturn res; + GstFlowReturn res = GST_FLOW_OK; GstSpeexDec *dec; + gboolean header_packet = FALSE; /* no fancy draining */ if (G_UNLIKELY (!buf)) @@ -498,42 +510,53 @@ gst_speex_dec_handle_frame (GstAudioDecoder * bdec, GstBuffer * buf) dec = GST_SPEEX_DEC (bdec); - /* If we have the streamheader and vorbiscomment from the caps already - * ignore them here */ - if (dec->streamheader && dec->vorbiscomment) { - if (memcmp_buffers (dec->streamheader, buf)) { - GST_DEBUG_OBJECT (dec, "found streamheader"); - gst_audio_decoder_finish_frame (bdec, NULL, 1); - res = GST_FLOW_OK; - } else if (memcmp_buffers (dec->vorbiscomment, buf)) { - GST_DEBUG_OBJECT (dec, "found vorbiscomments"); - gst_audio_decoder_finish_frame (bdec, NULL, 1); - res = GST_FLOW_OK; - } else { - res = gst_speex_dec_parse_data (dec, buf); - } - } else { - /* Otherwise fall back to packet counting and assume that the - * first two packets are the headers. */ - switch (dec->packetno) { - case 0: - GST_DEBUG_OBJECT (dec, "counted streamheader"); + switch (dec->packetno) { + case 0: + GST_DEBUG_OBJECT (dec, "expecting streamheader"); + header_packet = TRUE; + if (!memcmp_buffers (dec->streamheader, buf)) { res = gst_speex_dec_parse_header (dec, buf); - gst_audio_decoder_finish_frame (bdec, NULL, 1); - break; - case 1: - GST_DEBUG_OBJECT (dec, "counted vorbiscomments"); + if (res == GST_FLOW_NO_HEADER) { + header_packet = FALSE; + GST_INFO_OBJECT (dec, "No streamheader in first buffer"); + if (dec->streamheader == NULL) { + GST_ERROR_OBJECT (dec, "Can't proceed without a header"); + return GST_FLOW_ERROR; + } else { + GST_INFO_OBJECT (dec, "Using streamheader from caps"); + } + } + /* we prefer "inband" streamheaders to the ones in caps */ + if (res == GST_FLOW_OK) { + GST_DEBUG_OBJECT (dec, "found streamheader"); + gst_buffer_replace (&dec->streamheader, buf); + } + } + break; + case 1: + GST_DEBUG_OBJECT (dec, "expecting vorbiscomment"); + header_packet = TRUE; + if (!memcmp_buffers (dec->vorbiscomment, buf)) { res = gst_speex_dec_parse_comments (dec, buf); - gst_audio_decoder_finish_frame (bdec, NULL, 1); - break; - default: - { - res = gst_speex_dec_parse_data (dec, buf); - break; + if (res == GST_FLOW_NO_HEADER) { + header_packet = FALSE; + GST_INFO_OBJECT (dec, "No vorbisheader in second buffer"); + } + if (res == GST_FLOW_OK) { + GST_DEBUG_OBJECT (dec, "found vorbiscomment"); + gst_buffer_replace (&dec->vorbiscomment, buf); + } } - } + break; + default: + break; } + if (header_packet) + gst_audio_decoder_finish_frame (bdec, NULL, 1); + else + res = gst_speex_dec_parse_data (dec, buf); + dec->packetno++; return res; diff --git a/tests/check/elements/speex.c b/tests/check/elements/speex.c index 6cc3bb671ea..c568349b372 100644 --- a/tests/check/elements/speex.c +++ b/tests/check/elements/speex.c @@ -91,16 +91,125 @@ GST_START_TEST (test_encoder_to_decoder_timestamp) } GST_END_TEST; +GST_START_TEST (test_headers_in_caps) +{ + GstHarness * h = gst_harness_new ("speexdec"); + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\" ! " + "speexenc", TRUE); + + /* the first audio-buffer produces 3 packets: + streamheader, vorbiscomments and the encoded buffer */ + gst_harness_src_crank_and_push_many (h, 1, 3); + + /* verify the decoder produces exactly one decoded buffer */ + gst_buffer_unref (gst_harness_pull (h)); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_headers_in_buffers) +{ + GstHarness * h = gst_harness_new ("speexdec"); + gst_harness_set_src_caps_str (h, "audio/x-speex"); + + /* turn off forwarding to avoid getting caps from the src-harness, + and hence not accessing the streamheaders through the caps */ + gst_harness_set_forwarding (h, FALSE); + + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\" ! " + "speexenc", TRUE); + + /* the first audio-buffer produces 3 packets: + streamheader, vorbiscomments and the encoded buffer */ + gst_harness_src_crank_and_push_many (h, 1, 3); + + /* verify the decoder produces exactly one decoded buffer */ + gst_buffer_unref (gst_harness_pull (h)); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_headers_not_in_buffers) +{ + GstHarness * h = gst_harness_new ("speexdec"); + + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\" ! " + "speexenc", TRUE); + + /* the first audio-buffer produces 3 packets: + streamheader, vorbiscomments and the encoded buffer */ + gst_harness_src_crank_and_push_many (h, 1, 0); + + /* now remove the streamheader and vorbiscomment */ + gst_buffer_unref (gst_harness_pull (h->src_harness)); + gst_buffer_unref (gst_harness_pull (h->src_harness)); + + /* and just push the encoded buffer to the decoder */ + gst_harness_src_crank_and_push_many (h, 0, 1); + + /* verify the decoder produces exactly one decoded buffer */ + gst_buffer_unref (gst_harness_pull (h)); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_headers_from_flv) +{ + GstHarness * h = gst_harness_new ("speexdec"); + + /* set caps with the same streamheader flvdemux would add */ + gst_harness_set_src_caps_str (h, "audio/x-speex, " + "streamheader=(buffer)< 5370656578202020312e312e3132000000000000000000000" + "00000000100000050000000803e0000010000000400000001000000ffffffff500000000" + "000000001000000000000000000000000000000, " + "0b0000004e6f20636f6d6d656e74730000000001 >, " + "rate=(int)16000, channels=(int)1"); + + /* turn off forwarding to avoid getting caps from the src-harness, + and force it to use the flv-streamheaders we have set in the caps */ + gst_harness_set_forwarding (h, FALSE); + + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\" ! " + "speexenc", TRUE); + + /* the first audio-buffer produces 3 packets: + streamheader, vorbiscomments and the encoded buffer */ + gst_harness_src_crank_and_push_many (h, 1, 3); + + /* verify the decoder produces exactly one decoded buffer */ + gst_buffer_unref (gst_harness_pull (h)); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + gst_harness_teardown (h); +} +GST_END_TEST + static Suite * speex_suite (void) { Suite *s = suite_create ("speex"); - TCase *tc_chain = tcase_create ("speex"); + TCase *tc_chain; - suite_add_tcase (s, tc_chain); + suite_add_tcase (s, (tc_chain = tcase_create ("timestamping"))); tcase_add_test (tc_chain, test_encoder_timestamp); tcase_add_test (tc_chain, test_encoder_to_decoder_timestamp); + suite_add_tcase (s, (tc_chain = tcase_create ("headers"))); + tcase_add_test (tc_chain, test_headers_in_caps); + tcase_add_test (tc_chain, test_headers_in_buffers); + tcase_add_test (tc_chain, test_headers_not_in_buffers); + tcase_add_test (tc_chain, test_headers_from_flv); + return s; } From 8fbb8430d0c810c097c9c4bc6133d9d25f9b5d51 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 26 Sep 2016 14:12:19 +0200 Subject: [PATCH 136/339] speexenc: Fix segment before caps event when draining Don't send a segment event if we haven't produced data and set the output caps yet. Doing so causes a g_warning about sticky event misordering. https://bugzilla.gnome.org/show_bug.cgi?id=773510 --- .../gst-plugins-good/ext/speex/gstspeexenc.c | 3 +- tests/check/elements/speex.c | 45 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c b/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c index c834a10b69b..07b0311e4ad 100644 --- a/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c +++ b/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c @@ -559,7 +559,8 @@ gst_speex_enc_encode (GstSpeexEnc * enc, GstBuffer * buf) segment = &GST_AUDIO_ENCODER_OUTPUT_SEGMENT (enc); GST_DEBUG_OBJECT (enc, "existing output segment %" GST_SEGMENT_FORMAT, segment); - if (!GST_CLOCK_TIME_IS_VALID (segment->stop)) { + if (!GST_CLOCK_TIME_IS_VALID (segment->stop) && + gst_pad_has_current_caps (GST_AUDIO_ENCODER_SRC_PAD (enc))) { int input_samples = bsize / (enc->channels * 2); GST_DEBUG_OBJECT (enc, "No stop time and partial frame, updating segment"); diff --git a/tests/check/elements/speex.c b/tests/check/elements/speex.c index c568349b372..7db3cc2c9d7 100644 --- a/tests/check/elements/speex.c +++ b/tests/check/elements/speex.c @@ -194,6 +194,47 @@ GST_START_TEST (test_headers_from_flv) } GST_END_TEST +GST_START_TEST (test_new_segment_before_first_output_buffer) +{ + GstHarness *h = gst_harness_new ("speexenc"); + GstBuffer *buf; + GstSegment segment; + + /* 10 ms buffers */ + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=160 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\"", TRUE); + + /* push 10 ms of audio-data */ + gst_harness_push_from_src (h); + + /* send a second segment event */ + gst_segment_init (&segment, GST_FORMAT_TIME); + fail_unless (gst_harness_push_event (h, gst_event_new_segment (&segment))); + + /* push 10 ms of audio-data */ + gst_harness_push_from_src (h); + + /* pull 20 ms of encoded audio-data with headers */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} +GST_END_TEST; + + static Suite * speex_suite (void) { @@ -210,6 +251,10 @@ speex_suite (void) tcase_add_test (tc_chain, test_headers_not_in_buffers); tcase_add_test (tc_chain, test_headers_from_flv); + suite_add_tcase (s, (tc_chain = tcase_create ("events"))); + tcase_add_test (tc_chain, test_new_segment_before_first_output_buffer); + + return s; } From 957e9f81f8d020903eb35c8ee9957f6c14a282e6 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sat, 26 Mar 2016 11:04:44 +0100 Subject: [PATCH 137/339] flvdemux: Make "no-more-pads-threshold" a property Allows it to be configured differently for different use-cases, specifically the live scenario would probably never want this enabled. https://bugzilla.gnome.org/show_bug.cgi?id=773511 --- .../gst-plugins-good/gst/flv/gstflvdemux.c | 63 ++++++++++++++++--- .../gst-plugins-good/gst/flv/gstflvdemux.h | 3 + 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c index c4df091f079..480cf3fef9f 100644 --- a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c +++ b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c @@ -89,6 +89,12 @@ static GstStaticPadTemplate video_src_template = "video/x-h264, stream-format=avc;") ); +enum +{ + PROP_00, /* PROP_0 already used by included gstindex.c */ + PROP_NO_MORE_PADS_THRESHOLD +}; + #define gst_flv_demux_parent_class parent_class G_DEFINE_TYPE (GstFlvDemux, gst_flv_demux, GST_TYPE_ELEMENT); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (flvdemux, "flvdemux", @@ -103,7 +109,7 @@ GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (flvdemux, "flvdemux", #define RESYNC_THRESHOLD 2000 /* how much stream time to wait for audio tags to appear after we have video, or vice versa */ -#define NO_MORE_PADS_THRESHOLD (6 * GST_SECOND) +#define DEFAULT_NO_MORE_PADS_THRESHOLD (6 * GST_SECOND) static gboolean flv_demux_handle_seek_push (GstFlvDemux * demux, GstEvent * event); @@ -1396,9 +1402,10 @@ gst_flv_demux_parse_tag_audio (GstFlvDemux * demux, GstBuffer * buffer) demux->audio_first_ts = GST_BUFFER_TIMESTAMP (outbuf); } - if (G_UNLIKELY (!demux->streams_aware && !demux->no_more_pads - && (GST_CLOCK_DIFF (demux->audio_start, - GST_BUFFER_TIMESTAMP (outbuf)) > NO_MORE_PADS_THRESHOLD))) { + if (G_UNLIKELY (!demux->streams_aware && !demux->no_more_pads && + (demux->no_more_pads_threshold != -1) && + (GST_CLOCK_DIFF (demux->audio_start, + GST_BUFFER_TIMESTAMP (outbuf)) > demux->no_more_pads_threshold))) { GST_DEBUG_OBJECT (demux, "Signalling no-more-pads because no video stream was found" " after 6 seconds of audio"); @@ -1874,9 +1881,10 @@ gst_flv_demux_parse_tag_video (GstFlvDemux * demux, GstBuffer * buffer) demux->video_first_ts = GST_BUFFER_TIMESTAMP (outbuf); } - if (G_UNLIKELY (!demux->streams_aware && !demux->no_more_pads - && (GST_CLOCK_DIFF (demux->video_start, - GST_BUFFER_TIMESTAMP (outbuf)) > NO_MORE_PADS_THRESHOLD))) { + if (G_UNLIKELY (!demux->streams_aware && !demux->no_more_pads && + (demux->no_more_pads_threshold != -1) && + (GST_CLOCK_DIFF (demux->video_start, + GST_BUFFER_TIMESTAMP (outbuf)) > demux->no_more_pads_threshold))) { GST_DEBUG_OBJECT (demux, "Signalling no-more-pads because no audio stream was found" " after 6 seconds of video"); @@ -3800,6 +3808,38 @@ gst_flv_demux_get_index (GstElement * element) return result; } +static void +gst_flv_demux_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstFlvDemux *demux = GST_FLV_DEMUX (object); + + switch (prop_id) { + case PROP_NO_MORE_PADS_THRESHOLD: + g_value_set_int64 (value, demux->no_more_pads_threshold); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_flv_demux_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstFlvDemux *demux = GST_FLV_DEMUX (object); + + switch (prop_id) { + case PROP_NO_MORE_PADS_THRESHOLD: + demux->no_more_pads_threshold = g_value_get_int64 (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + static void gst_flv_demux_dispose (GObject * object) { @@ -3882,6 +3922,8 @@ gst_flv_demux_class_init (GstFlvDemuxClass * klass) GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass); GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + gobject_class->get_property = gst_flv_demux_get_property; + gobject_class->set_property = gst_flv_demux_set_property; gobject_class->dispose = gst_flv_demux_dispose; gstelement_class->change_state = @@ -3901,6 +3943,13 @@ gst_flv_demux_class_init (GstFlvDemuxClass * klass) gst_element_class_set_static_metadata (gstelement_class, "FLV Demuxer", "Codec/Demuxer", "Demux FLV feeds into digital streams", "Julien Moutte "); + + g_object_class_install_property (gobject_class, PROP_NO_MORE_PADS_THRESHOLD, + g_param_spec_int64 ("no-more-pads-threshold", "No-More-Pads Threshold", + "How much stream time to wait for audio tags to appear after we " + "have video, or vice versa (-1 waits forever)", + -1, G_MAXINT64, DEFAULT_NO_MORE_PADS_THRESHOLD, + G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } static void diff --git a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.h b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.h index 71584577177..d04332229f7 100644 --- a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.h +++ b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.h @@ -162,6 +162,9 @@ struct _GstFlvDemux gboolean audio_done; gint64 from_offset; gint64 to_offset; + + /* properties */ + gint64 no_more_pads_threshold; }; struct _GstFlvDemuxClass From df02d6c445931b865dee158c474a47205f7a3037 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sat, 26 Mar 2016 11:06:33 +0100 Subject: [PATCH 138/339] flvdemux: Set no-more-pads-threshold default to -1 [pexhack] This is a Pexip specific change since we are only interested in using gstflvdemux live. --- subprojects/gst-plugins-good/gst/flv/gstflvdemux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c index 480cf3fef9f..665b538cfa5 100644 --- a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c +++ b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c @@ -109,7 +109,7 @@ GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (flvdemux, "flvdemux", #define RESYNC_THRESHOLD 2000 /* how much stream time to wait for audio tags to appear after we have video, or vice versa */ -#define DEFAULT_NO_MORE_PADS_THRESHOLD (6 * GST_SECOND) +#define DEFAULT_NO_MORE_PADS_THRESHOLD -1 static gboolean flv_demux_handle_seek_push (GstFlvDemux * demux, GstEvent * event); From a059176c9656ef029ed4383f65b976ef1b8bd081 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 8 Feb 2019 03:53:43 +0100 Subject: [PATCH 139/339] flvmux: [pexhack] lower backwards-dts warning to info This happens all the time for us, so no point warning about it. --- subprojects/gst-plugins-good/gst/flv/gstflvmux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/flv/gstflvmux.c b/subprojects/gst-plugins-good/gst/flv/gstflvmux.c index 37f7d6d5a39..304c7f64de5 100644 --- a/subprojects/gst-plugins-good/gst/flv/gstflvmux.c +++ b/subprojects/gst-plugins-good/gst/flv/gstflvmux.c @@ -1284,7 +1284,7 @@ gst_flv_mux_buffer_to_tag_internal (GstFlvMux * mux, GstBuffer * buffer, * also between the audio & video streams. */ if (dts < mux->last_dts && mux->enforce_increasing_timestamps) { - GST_WARNING_OBJECT (pad, "Got backwards dts! (%" GST_TIME_FORMAT + GST_INFO_OBJECT (pad, "Got backwards dts! (%" GST_TIME_FORMAT " < %" GST_TIME_FORMAT ")", GST_TIME_ARGS (dts * GST_MSECOND), GST_TIME_ARGS (mux->last_dts * GST_MSECOND)); dts = mux->last_dts; From af2af89251b13606819300215c4778d8a37b0b61 Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Mon, 30 Jun 2014 16:35:28 +0200 Subject: [PATCH 140/339] rtpmux: Limit sequence number to half G_MAXUINT16 [pexhack] --- subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c index a5ad2306cb3..89ca7cdfde6 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c @@ -967,7 +967,7 @@ gst_rtp_mux_ready_to_paused (GstRTPMux * rtp_mux) rtp_mux->send_stream_start = TRUE; if (rtp_mux->seqnum_offset == -1) - rtp_mux->seqnum_base = g_random_int_range (0, G_MAXUINT16); + rtp_mux->seqnum_base = g_random_int_range (0, G_MAXUINT16 / 2); else rtp_mux->seqnum_base = rtp_mux->seqnum_offset; rtp_mux->seqnum = rtp_mux->seqnum_base; From 31e411a939e2299e5e3489661a70133917e385a4 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 12 Feb 2020 14:28:38 +0100 Subject: [PATCH 141/339] rtpmux: fix test warnings --- .../gst-plugins-good/tests/check/elements/rtpmux.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpmux.c b/subprojects/gst-plugins-good/tests/check/elements/rtpmux.c index c70018f22eb..aa61c589d5e 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpmux.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpmux.c @@ -40,7 +40,7 @@ static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src", typedef void (*check_cb) (GstPad * pad, int i); static gboolean -query_func (GstPad * pad, GstObject * noparent, GstQuery * query) +query_func (GstPad * pad, G_GNUC_UNUSED GstObject * noparent, GstQuery * query) { switch (GST_QUERY_TYPE (query)) { case GST_QUERY_CAPS: @@ -71,7 +71,7 @@ remove_ssrc_from_caps (GstCaps * caps) } static gboolean -event_func (GstPad * pad, GstObject * noparent, GstEvent * event) +event_func (GstPad * pad, G_GNUC_UNUSED GstObject * noparent, GstEvent * event) { switch (GST_EVENT_TYPE (event)) { case GST_EVENT_CAPS: @@ -208,8 +208,7 @@ test_basic (const gchar * elem_name, const gchar * sink2, int count, fail_unless (gst_pad_push (src1, inbuf) == GST_FLOW_OK); if (buffers) - fail_unless (GST_BUFFER_PTS (buffers->data) == i * 1000, "%lld", - GST_BUFFER_PTS (buffers->data)); + fail_unless_equals_uint64 (GST_BUFFER_PTS (buffers->data), i * 1000); cb (src2, i); @@ -241,7 +240,7 @@ test_basic (const gchar * elem_name, const gchar * sink2, int count, } static void -basic_check_cb (GstPad * pad, int i) +basic_check_cb (G_GNUC_UNUSED GstPad * pad, int i) { GstRTPBuffer rtpbuffer = GST_RTP_BUFFER_INIT; fail_unless (buffers && g_list_length (buffers) == 1); From 650047f5c9b978b88d34d10b1a21b6b403353a50 Mon Sep 17 00:00:00 2001 From: Mikhail Fludkov Date: Wed, 14 Sep 2016 17:37:45 +0200 Subject: [PATCH 142/339] rtpssrcdemux: avoid the log spam [pexhack] --- .../gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c | 8 +++++--- .../gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c index 243449be28f..3ac25327014 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c @@ -731,9 +731,11 @@ gst_rtp_ssrc_demux_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) create_failed: { gst_buffer_unref (buf); - GST_WARNING_OBJECT (demux, - "Dropping buffer SSRC %08x. " - "Max streams number reached (%u)", ssrc, demux->max_streams); + if ((demux->err_num & 0xff) == 0) + GST_WARNING_OBJECT (demux, + "Dropping buffer SSRC %08x. " + "Max streams number reached (%u)", ssrc, demux->max_streams); + ++demux->err_num; return GST_FLOW_OK; } } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h index e7e347d251d..fecfded69de 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h @@ -41,6 +41,7 @@ struct _GstRtpSsrcDemux GRecMutex padlock; GSList *srcpads; guint max_streams; + guint err_num; }; struct _GstRtpSsrcDemuxClass From 402430cb97f0838963ce67bdb01c7fdd8e54ddba Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 29 May 2015 13:58:18 +0200 Subject: [PATCH 143/339] vpx: Build SIMD-ified vpx plugin if available [pexhack] Multiple SIMD-ified vpx plugins will be built if available. These plugins will be put in a separate plugin directory which can be linked to if configuring with --enable-simd-symlink=avx etc. --- .../gst-plugins-good/ext/vpx/meson.build | 42 ++++++++++++++++++- subprojects/gst-plugins-good/ext/vpx/plugin.c | 25 ++++++++--- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/meson.build b/subprojects/gst-plugins-good/ext/vpx/meson.build index 9b8b34c67e3..4199b235048 100644 --- a/subprojects/gst-plugins-good/ext/vpx/meson.build +++ b/subprojects/gst-plugins-good/ext/vpx/meson.build @@ -18,7 +18,7 @@ vpx_features = [ ] vpx_option = get_option('vpx') -vpx_dep = dependency('vpx', version : '>=1.7.0', required : vpx_option, allow_fallback: true) +vpx_dep = dependency('vpx-avx', version : '>=1.5.0', required : vpx_option) if vpx_dep.found() vpx_args = [] @@ -93,3 +93,43 @@ if vpx_dep.found() env.prepend('GST_PRESET_PATH', meson.current_source_dir()) meson.add_devenv(env) endif + + + + +#Pexip spesific + +pex_isas = [ + ['avx', ['-DGST_ISA_AVX'] ], + ['avx2', ['-DGST_ISA_AVX2'] ], + ['avx512', ['-DGST_ISA_AVX512'] ], +] + +vpx_args = [ + '-DHAVE_VPX_1_3', + '-DHAVE_VPX_1_4', + '-DHAVE_VP8_ENCODER', + '-DHAVE_VP8_DECODER', + '-DHAVE_VP9_ENCODER', + '-DHAVE_VP9_DECODER', +] + +plugins_install_dir_isa = '@0@-'.format(plugins_install_dir) + +foreach isa_flags : pex_isas + isa = isa_flags[0] + flags = isa_flags[1] + + vpxisa_dep = dependency('vpx-@0@'.format(isa), required : false) + + if vpxisa_dep.found() + gstvpxisa = shared_library('gstvpx_' + isa, + vpx_sources, gstvpx_enums, + c_args : gst_plugins_good_args + vpx_args + flags, + include_directories: [configinc], + dependencies : [gstbase_dep, gsttag_dep, gstvideo_dep, vpxisa_dep], + install : true, + install_dir : plugins_install_dir_isa + isa, + ) + endif +endforeach diff --git a/subprojects/gst-plugins-good/ext/vpx/plugin.c b/subprojects/gst-plugins-good/ext/vpx/plugin.c index 1887aee70e2..e1472c550e2 100644 --- a/subprojects/gst-plugins-good/ext/vpx/plugin.c +++ b/subprojects/gst-plugins-good/ext/vpx/plugin.c @@ -33,26 +33,41 @@ plugin_init (GstPlugin * plugin) gboolean ret = FALSE; #ifdef HAVE_VP8_DECODER - ret |= GST_ELEMENT_REGISTER (vp8dec, plugin); + if (!g_type_from_name ("GstVP8Dec")) + ret |= GST_ELEMENT_REGISTER (vp8dec, plugin); #endif #ifdef HAVE_VP8_ENCODER - ret |= GST_ELEMENT_REGISTER (vp8enc, plugin); + if (!g_type_from_name ("GstVP8Enc")) + ret |= GST_ELEMENT_REGISTER (vp8enc, plugin); #endif #ifdef HAVE_VP9_DECODER - ret |= GST_ELEMENT_REGISTER (vp9dec, plugin); + if (!g_type_from_name ("GstVP9Dec")) + ret |= GST_ELEMENT_REGISTER (vp9dec, plugin); #endif #ifdef HAVE_VP9_ENCODER - ret |= GST_ELEMENT_REGISTER (vp9enc, plugin); + if (!g_type_from_name ("GstVP9Enc")) + ret |= GST_ELEMENT_REGISTER (vp9enc, plugin); #endif return ret; } +/* Pexip Spesific */ +#if defined (GST_ISA_AVX) +# define GST_ISA_SUFFIX(name) G_PASTE (name, _avx) +#elif defined (GST_ISA_AVX2) +# define GST_ISA_SUFFIX(name) G_PASTE (name, _avx2) +#elif defined (GST_ISA_AVX512) +# define GST_ISA_SUFFIX(name) G_PASTE (name, _avx512) +#else +# define GST_ISA_SUFFIX(name) name +#endif + GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, - vpx, + GST_ISA_SUFFIX (vpx), "VP8 plugin", plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) From d88a53b780d6c297b98108647376b0c867bdf483 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 23 Jun 2014 11:24:38 +0200 Subject: [PATCH 144/339] vpxenc: Enable rate limit force keyframes events [pexhack] Hard code limit to 1500 ms between each generated key frame. This should ideally be a property and set to a value derived from RTT. This avoids generating bursts of keyframes when receiving many PLIs/NACKs in close succession. 1500ms is quite large, but with 500ms and lots of packetloss, the frequent keyframe generation will more then double the bitrate from the libvpx encoder (configured at 1mbps, producing over 2mbps), potentially making an already bad situation much worse. By adding another second we only see about 1,4mbps, which is more acceptable. --- subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c index 2676d898098..4364281faa9 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c @@ -839,6 +839,9 @@ gst_vpx_enc_init (GstVPXEnc * gst_vpx_enc) gst_vpx_enc->cfg.g_profile = DEFAULT_PROFILE; + gst_video_encoder_set_min_force_key_unit_interval ( + GST_VIDEO_ENCODER (gst_vpx_enc), 1500 * GST_MSECOND); + g_mutex_init (&gst_vpx_enc->encoder_lock); } From 3dbfdc4ee4be3cf7a2a3b4f32e49a49d5708b516 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 12 May 2017 15:51:54 +0200 Subject: [PATCH 145/339] vpxenc: Add tile-columns and tile-rows properties --- .../gst-plugins-good/ext/vpx/gstvp8enc.c | 8 ++ .../gst-plugins-good/ext/vpx/gstvp9enc.c | 8 ++ .../gst-plugins-good/ext/vpx/gstvpxenc.c | 79 +++++++++++++++++++ .../gst-plugins-good/ext/vpx/gstvpxenc.h | 4 + .../gst-plugins-good/ext/vpx/meson.build | 2 +- 5 files changed, 100 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp8enc.c b/subprojects/gst-plugins-good/ext/vpx/gstvp8enc.c index 5222c970a1f..0a08ec20830 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp8enc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp8enc.c @@ -98,6 +98,7 @@ gst_vp8_enc_user_data_free (GstVP8EncUserData * user_data) static vpx_codec_iface_t *gst_vp8_enc_get_algo (GstVPXEnc * enc); static gboolean gst_vp8_enc_enable_scaling (GstVPXEnc * enc); +static gboolean gst_vp8_enc_enable_tiles (GstVPXEnc * enc); static void gst_vp8_enc_set_image_format (GstVPXEnc * enc, vpx_image_t * image); static GstCaps *gst_vp8_enc_get_new_simple_caps (GstVPXEnc * enc); static void gst_vp8_enc_set_stream_info (GstVPXEnc * enc, GstCaps * caps, @@ -170,6 +171,7 @@ gst_vp8_enc_class_init (GstVP8EncClass * klass) vpx_encoder_class->get_algo = gst_vp8_enc_get_algo; vpx_encoder_class->enable_scaling = gst_vp8_enc_enable_scaling; + vpx_encoder_class->enable_tiles = gst_vp8_enc_enable_tiles; vpx_encoder_class->set_image_format = gst_vp8_enc_set_image_format; vpx_encoder_class->get_new_vpx_caps = gst_vp8_enc_get_new_simple_caps; vpx_encoder_class->set_stream_info = gst_vp8_enc_set_stream_info; @@ -218,6 +220,12 @@ gst_vp8_enc_enable_scaling (GstVPXEnc * enc) return TRUE; } +static gboolean +gst_vp8_enc_enable_tiles (GstVPXEnc * enc) +{ + return FALSE; +} + static void gst_vp8_enc_set_image_format (GstVPXEnc * enc, vpx_image_t * image) { diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c index e966e000c68..a9a94626be6 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c @@ -105,6 +105,7 @@ GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vp9enc, "vp9enc", GST_RANK_PRIMARY, static vpx_codec_iface_t *gst_vp9_enc_get_algo (GstVPXEnc * enc); static gboolean gst_vp9_enc_enable_scaling (GstVPXEnc * enc); +static gboolean gst_vp9_enc_enable_tiles (GstVPXEnc * enc); static void gst_vp9_enc_set_image_format (GstVPXEnc * enc, vpx_image_t * image); static GstCaps *gst_vp9_enc_get_new_simple_caps (GstVPXEnc * enc); static void gst_vp9_enc_set_stream_info (GstVPXEnc * enc, GstCaps * caps, @@ -233,6 +234,7 @@ gst_vp9_enc_class_init (GstVP9EncClass * klass) vpx_encoder_class->get_algo = gst_vp9_enc_get_algo; vpx_encoder_class->enable_scaling = gst_vp9_enc_enable_scaling; + vpx_encoder_class->enable_tiles = gst_vp9_enc_enable_tiles; vpx_encoder_class->set_image_format = gst_vp9_enc_set_image_format; vpx_encoder_class->get_new_vpx_caps = gst_vp9_enc_get_new_simple_caps; vpx_encoder_class->set_stream_info = gst_vp9_enc_set_stream_info; @@ -531,6 +533,12 @@ gst_vp9_enc_enable_scaling (GstVPXEnc * enc) return FALSE; } +static gboolean +gst_vp9_enc_enable_tiles (GstVPXEnc * enc) +{ + return TRUE; +} + static void gst_vp9_enc_set_image_format (GstVPXEnc * enc, vpx_image_t * image) { diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c index 4364281faa9..18b254f4714 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c @@ -82,6 +82,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_vpxenc_debug); #define DEFAULT_LAG_IN_FRAMES 0 #define DEFAULT_THREADS 0 +#define DEFAULT_TILE_COLUMNS 6 +#define DEFAULT_TILE_ROWS 0 #define DEFAULT_H_SCALING_MODE VP8E_NORMAL #define DEFAULT_V_SCALING_MODE VP8E_NORMAL @@ -139,6 +141,8 @@ enum PROP_ERROR_RESILIENT, PROP_LAG_IN_FRAMES, PROP_THREADS, + PROP_TILE_COLUMNS, + PROP_TILE_ROWS, PROP_DEADLINE, PROP_H_SCALING_MODE, PROP_V_SCALING_MODE, @@ -641,6 +645,20 @@ gst_vpx_enc_class_init (GstVPXEncClass * klass) (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_DOC_SHOW_DEFAULT))); +#ifdef HAVE_VPX_1_3 + g_object_class_install_property (gobject_class, PROP_TILE_COLUMNS, + g_param_spec_int ("tile-columns", "Tile Columns", + "Number of tile columns, log2", + 0, 6, DEFAULT_TILE_COLUMNS, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_TILE_ROWS, + g_param_spec_int ("tile-rows", "Tile Rows", + "Number of tile rows, log2", + 0, 2, DEFAULT_TILE_ROWS, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); +#endif + g_object_class_install_property (gobject_class, PROP_DEADLINE, g_param_spec_int64 ("deadline", "Deadline", "Deadline per frame (usec, 0=best, 1=realtime)", @@ -816,6 +834,8 @@ gst_vpx_enc_init (GstVPXEnc * gst_vpx_enc) gst_vpx_enc->cfg.g_error_resilient = DEFAULT_ERROR_RESILIENT; gst_vpx_enc->cfg.g_lag_in_frames = DEFAULT_LAG_IN_FRAMES; gst_vpx_enc->cfg.g_threads = DEFAULT_THREADS; + gst_vpx_enc->tile_columns = DEFAULT_TILE_COLUMNS; + gst_vpx_enc->tile_rows = DEFAULT_TILE_ROWS; gst_vpx_enc->deadline = DEFAULT_DEADLINE; gst_vpx_enc->h_scaling_mode = DEFAULT_H_SCALING_MODE; gst_vpx_enc->v_scaling_mode = DEFAULT_V_SCALING_MODE; @@ -914,12 +934,15 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, GstVPXEnc *gst_vpx_enc; gboolean global = FALSE; vpx_codec_err_t status; + GstVPXEncClass *gst_vpx_enc_class; g_return_if_fail (GST_IS_VPX_ENC (object)); gst_vpx_enc = GST_VPX_ENC (object); + gst_vpx_enc_class = GST_VPX_ENC_GET_CLASS (gst_vpx_enc); GST_DEBUG_OBJECT (object, "gst_vpx_enc_set_property"); g_mutex_lock (&gst_vpx_enc->encoder_lock); + switch (prop_id) { case PROP_RC_END_USAGE: gst_vpx_enc->cfg.rc_end_usage = g_value_get_enum (value); @@ -1130,6 +1153,34 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, gst_vpx_enc->cfg.g_threads = g_value_get_int (value); global = TRUE; break; +#ifdef HAVE_VPX_1_3 + case PROP_TILE_COLUMNS: + gst_vpx_enc->tile_columns = g_value_get_int (value); + if (gst_vpx_enc_class->enable_tiles (gst_vpx_enc) && gst_vpx_enc->inited) { + status = + vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_TILE_COLUMNS, + gst_vpx_enc->tile_columns); + if (status != VPX_CODEC_OK) { + GST_WARNING_OBJECT (gst_vpx_enc, + "Failed to set VP9E_SET_TILE_COLUMNS: %s", + gst_vpx_error_name (status)); + } + } + break; + case PROP_TILE_ROWS: + gst_vpx_enc->tile_rows = g_value_get_int (value); + if (gst_vpx_enc_class->enable_tiles (gst_vpx_enc) && gst_vpx_enc->inited) { + status = + vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_TILE_ROWS, + gst_vpx_enc->tile_rows); + if (status != VPX_CODEC_OK) { + GST_WARNING_OBJECT (gst_vpx_enc, + "Failed to set VP9E_SET_TILE_ROWS: %s", + gst_vpx_error_name (status)); + } + } + break; +#endif case PROP_DEADLINE: gst_vpx_enc->deadline = g_value_get_int64 (value); break; @@ -1519,6 +1570,14 @@ gst_vpx_enc_get_property (GObject * object, guint prop_id, GValue * value, case PROP_THREADS: g_value_set_int (value, gst_vpx_enc->cfg.g_threads); break; +#ifdef HAVE_VPX_1_3 + case PROP_TILE_COLUMNS: + g_value_set_int (value, gst_vpx_enc->tile_columns); + break; + case PROP_TILE_ROWS: + g_value_set_int (value, gst_vpx_enc->tile_rows); + break; +#endif case PROP_DEADLINE: g_value_set_int64 (value, gst_vpx_enc->deadline); break; @@ -1851,6 +1910,26 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, } } +#ifdef HAVE_VPX_1_3 + if (vpx_enc_class->enable_tiles (encoder)) { + status = + vpx_codec_control (&encoder->encoder, VP9E_SET_TILE_COLUMNS, + encoder->tile_columns); + if (status != VPX_CODEC_OK) { + GST_WARNING_OBJECT (encoder, "Failed to set VP9E_SET_TILE_COLUMNS: %s", + gst_vpx_error_name (status)); + } + + status = + vpx_codec_control (&encoder->encoder, VP9E_SET_TILE_ROWS, + encoder->tile_rows); + if (status != VPX_CODEC_OK) { + GST_WARNING_OBJECT (encoder, "Failed to set VP9E_SET_TILE_ROWS: %s", + gst_vpx_error_name (status)); + } + } +#endif + status = vpx_codec_control (&encoder->encoder, VP8E_SET_CPUUSED, encoder->cpu_used); diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.h b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.h index d9c40e5440e..188b1b42f46 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.h +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.h @@ -87,6 +87,8 @@ struct _GstVPXEnc gint64 deadline; /* Controls */ + unsigned int tile_columns; + unsigned int tile_rows; VPX_SCALING_MODE h_scaling_mode; VPX_SCALING_MODE v_scaling_mode; int cpu_used; @@ -132,6 +134,8 @@ struct _GstVPXEncClass gboolean (*enable_scaling) (GstVPXEnc *enc); /*called from set_format with lock taken*/ gboolean (*configure_encoder) (GstVPXEnc *enc, GstVideoCodecState *state); + /*enabled tiles*/ + gboolean (*enable_tiles) (GstVPXEnc *enc); /*set image format info*/ void (*set_image_format) (GstVPXEnc *enc, vpx_image_t *image); /*get new simple caps*/ diff --git a/subprojects/gst-plugins-good/ext/vpx/meson.build b/subprojects/gst-plugins-good/ext/vpx/meson.build index 4199b235048..096204f5285 100644 --- a/subprojects/gst-plugins-good/ext/vpx/meson.build +++ b/subprojects/gst-plugins-good/ext/vpx/meson.build @@ -21,7 +21,7 @@ vpx_option = get_option('vpx') vpx_dep = dependency('vpx-avx', version : '>=1.5.0', required : vpx_option) if vpx_dep.found() - vpx_args = [] + vpx_args = ['-DHAVE_VPX_1_3'] foreach f : vpx_features header = f.get(0) codec_iface = f.get(1) From ec95602db0db9968a215e65fe445513ff8a4c283 Mon Sep 17 00:00:00 2001 From: "sergei.kovalev" Date: Tue, 7 Jul 2020 18:59:34 +0200 Subject: [PATCH 146/339] gstvpxenc: fix auto bitrate functionality v2 --- .../gst-plugins-good/ext/vpx/gstvpxenc.c | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c index 18b254f4714..d17219dc666 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c @@ -799,7 +799,7 @@ gst_vpx_enc_init (GstVPXEnc * gst_vpx_enc) gst_vpx_enc->cfg.rc_end_usage = DEFAULT_RC_END_USAGE; gst_vpx_enc->cfg.rc_target_bitrate = DEFAULT_RC_TARGET_BITRATE / 1000; - gst_vpx_enc->rc_target_bitrate_auto = DEFAULT_RC_TARGET_BITRATE == 0; + gst_vpx_enc->rc_target_bitrate_auto = FALSE; gst_vpx_enc->cfg.rc_min_quantizer = DEFAULT_RC_MIN_QUANTIZER; gst_vpx_enc->cfg.rc_max_quantizer = DEFAULT_RC_MAX_QUANTIZER; gst_vpx_enc->cfg.rc_dropframe_thresh = DEFAULT_RC_DROPFRAME_THRESH; @@ -892,39 +892,41 @@ gst_vpx_enc_finalize (GObject * object) } static void -gst_vpx_enc_set_auto_bitrate (GstVPXEnc * encoder) +gst_vpx_enc_set_auto_bitrate (GstVPXEnc * encoder, + GstVideoCodecState * input_state) { - if (encoder->input_state != NULL) { - guint size; - guint pixels_per_sec; - guint target_bitrate; - guint fps_n, fps_d; - - if (GST_VIDEO_INFO_FPS_N (&encoder->input_state->info) != 0) { - fps_n = GST_VIDEO_INFO_FPS_N (&encoder->input_state->info); - fps_d = GST_VIDEO_INFO_FPS_D (&encoder->input_state->info); - } else { - /* otherwise assume 30 frames per second as a fallback */ - fps_n = 30; - fps_d = 1; - } + guint size; + guint pixels_per_sec; + guint target_bitrate; + guint fps_n, fps_d; + + if (input_state == NULL) + return; - size = - GST_VIDEO_INFO_WIDTH (&encoder->input_state->info) * - GST_VIDEO_INFO_HEIGHT (&encoder->input_state->info); - pixels_per_sec = size * fps_n / fps_d; - target_bitrate = pixels_per_sec * encoder->bits_per_pixel; - - GST_DEBUG_OBJECT (encoder, - "Setting autobitrate for %ux%ux @ %u/%ufps %.4f = %ubps", - GST_VIDEO_INFO_WIDTH (&encoder->input_state->info), - GST_VIDEO_INFO_HEIGHT (&encoder->input_state->info), - GST_VIDEO_INFO_FPS_N (&encoder->input_state->info), - GST_VIDEO_INFO_FPS_D (&encoder->input_state->info), - encoder->bits_per_pixel, target_bitrate); - - encoder->cfg.rc_target_bitrate = target_bitrate / 1000; + if (GST_VIDEO_INFO_FPS_D (&input_state->info) != 0) { + fps_n = GST_VIDEO_INFO_FPS_N (&input_state->info); + fps_d = GST_VIDEO_INFO_FPS_D (&input_state->info); + } else { + /* otherwise assume 30 frames per second as a fallback */ + fps_n = 30; + fps_d = 1; } + + size = + GST_VIDEO_INFO_WIDTH (&input_state->info) * + GST_VIDEO_INFO_HEIGHT (&input_state->info); + pixels_per_sec = size * fps_n / fps_d; + target_bitrate = pixels_per_sec * encoder->bits_per_pixel; + + GST_DEBUG_OBJECT (encoder, + "Setting autobitrate for %ux%ux @ %u/%ufps %.4f = %ubps", + GST_VIDEO_INFO_WIDTH (&input_state->info), + GST_VIDEO_INFO_HEIGHT (&input_state->info), + GST_VIDEO_INFO_FPS_N (&input_state->info), + GST_VIDEO_INFO_FPS_D (&input_state->info), + encoder->bits_per_pixel, target_bitrate); + + encoder->cfg.rc_target_bitrate = target_bitrate / 1000; } static void @@ -950,7 +952,7 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, break; case PROP_RC_TARGET_BITRATE: if (g_value_get_int (value) == 0) { - gst_vpx_enc_set_auto_bitrate (gst_vpx_enc); + gst_vpx_enc_set_auto_bitrate (gst_vpx_enc, gst_vpx_enc->input_state); gst_vpx_enc->rc_target_bitrate_auto = TRUE; } else { gst_vpx_enc->cfg.rc_target_bitrate = g_value_get_int (value) / 1000; @@ -1368,7 +1370,7 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, case PROP_BITS_PER_PIXEL: gst_vpx_enc->bits_per_pixel = g_value_get_float (value); if (gst_vpx_enc->rc_target_bitrate_auto) { - gst_vpx_enc_set_auto_bitrate (gst_vpx_enc); + gst_vpx_enc_set_auto_bitrate (gst_vpx_enc, gst_vpx_enc->input_state); global = TRUE; } break; @@ -1827,6 +1829,10 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, encoder->cfg.g_w = GST_VIDEO_INFO_WIDTH (info); encoder->cfg.g_h = GST_VIDEO_INFO_HEIGHT (info); + /* Scale default bitrate to our size */ + if (encoder->rc_target_bitrate_auto) + gst_vpx_enc_set_auto_bitrate (encoder, state); + if (encoder->timebase_n != 0 && encoder->timebase_d != 0) { GST_DEBUG_OBJECT (video_encoder, "Using timebase configuration"); encoder->cfg.g_timebase.num = encoder->timebase_n; @@ -2032,10 +2038,6 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, gst_video_codec_state_unref (encoder->input_state); encoder->input_state = gst_video_codec_state_ref (state); - /* Scale default bitrate to our size */ - if (encoder->rc_target_bitrate_auto) - gst_vpx_enc_set_auto_bitrate (encoder); - /* prepare cached image buffer setup */ image = &encoder->image; memset (image, 0, sizeof (*image)); From 94d39d2c134cffc25194758d520ebbe8b8c57521 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Fri, 4 Nov 2016 10:33:24 +0100 Subject: [PATCH 147/339] vpxdec: Add property "direct-rendering" Adds property that can be used to disable direct rendering. --- .../gst-plugins-good/ext/vpx/gstvpxdec.c | 24 +++++++++++++++---- .../gst-plugins-good/ext/vpx/gstvpxdec.h | 1 + 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c index d5351b839a4..19434c32198 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c @@ -40,6 +40,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_vpxdec_debug); #define DEFAULT_DEBLOCKING_LEVEL 4 #define DEFAULT_NOISE_LEVEL 0 #define DEFAULT_THREADS 0 +#define DEFAULT_DIRECT_RENDERING TRUE #define DEFAULT_VIDEO_CODEC_TAG NULL #define DEFAULT_CODEC_ALGO NULL @@ -50,7 +51,8 @@ enum PROP_POST_PROCESSING_FLAGS, PROP_DEBLOCKING_LEVEL, PROP_NOISE_LEVEL, - PROP_THREADS + PROP_THREADS, + PROP_DIRECT_RENDERING }; #define C_FLAGS(v) ((guint) v) @@ -167,6 +169,11 @@ gst_vpx_dec_class_init (GstVPXDecClass * klass) "Maximum number of decoding threads", 0, 16, DEFAULT_THREADS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_DIRECT_RENDERING, + g_param_spec_boolean ("direct-rendering", "Direct Rendering", + "Enable direct rendering", DEFAULT_DIRECT_RENDERING, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + base_video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_vpx_dec_start); base_video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vpx_dec_stop); base_video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_vpx_dec_flush); @@ -205,6 +212,7 @@ gst_vpx_dec_init (GstVPXDec * gst_vpx_dec) gst_vpx_dec->post_processing_flags = DEFAULT_POST_PROCESSING_FLAGS; gst_vpx_dec->deblocking_level = DEFAULT_DEBLOCKING_LEVEL; gst_vpx_dec->noise_level = DEFAULT_NOISE_LEVEL; + gst_vpx_dec->direct_rendering = DEFAULT_DIRECT_RENDERING; if (vpxclass->get_needs_sync_point) { gst_video_decoder_set_needs_sync_point (GST_VIDEO_DECODER (gst_vpx_dec), @@ -242,6 +250,9 @@ gst_vpx_dec_set_property (GObject * object, guint prop_id, case PROP_THREADS: dec->threads = g_value_get_uint (value); break; + case PROP_DIRECT_RENDERING: + dec->direct_rendering = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -273,6 +284,9 @@ gst_vpx_dec_get_property (GObject * object, guint prop_id, GValue * value, case PROP_THREADS: g_value_set_uint (value, dec->threads); break; + case PROP_DIRECT_RENDERING: + g_value_set_boolean (value, dec->direct_rendering); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -658,8 +672,10 @@ gst_vpx_dec_open_codec (GstVPXDec * dec, GstVideoCodecFrame * frame) gst_vpx_error_name (status)); } } - vpx_codec_set_frame_buffer_functions (&dec->decoder, - gst_vpx_dec_get_buffer_cb, gst_vpx_dec_release_buffer_cb, dec); + if (dec->direct_rendering) { + vpx_codec_set_frame_buffer_functions (&dec->decoder, + gst_vpx_dec_get_buffer_cb, gst_vpx_dec_release_buffer_cb, dec); + } dec->decoder_inited = TRUE; @@ -753,7 +769,7 @@ gst_vpx_dec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame) gst_video_decoder_drop_frame (decoder, frame); } else { gst_vpx_dec_handle_resolution_change (dec, img, fmt); - if (img->fb_priv && dec->have_video_meta) { + if (dec->direct_rendering && img->fb_priv && dec->have_video_meta) { frame->output_buffer = gst_vpx_dec_prepare_image (dec, img); ret = gst_video_decoder_finish_frame (decoder, frame); } else { diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.h b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.h index e48f4e71432..3c73314aa6b 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.h +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.h @@ -75,6 +75,7 @@ struct _GstVPXDec gint deblocking_level; gint noise_level; gint threads; + gboolean direct_rendering; GstVideoCodecState *input_state; GstVideoCodecState *output_state; From f17b0746f67444ea49272eee13928ec2c8fe94fa Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 22 May 2019 11:16:56 +0200 Subject: [PATCH 148/339] vpxdec: Fix direct rendering, don't hold write access When buffer is pushed downstream, we should not hold the buffer mapped with write access. Doing so would often lead to unneccesary memcpy later. For instance, gst_buffer_make_writable() in gst_video_decoder_finish_frame() will cause a memcpy because of _memory_get_exclusive_reference(). --- subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c index 19434c32198..f9a189eef9a 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c @@ -415,6 +415,8 @@ gst_vpx_dec_prepare_image (GstVPXDec * dec, const vpx_image_t * img) GstVideoInfo *info = &dec->output_state->info; buffer = gst_buffer_ref (frame->buffer); + gst_buffer_unmap (buffer, &frame->info); + gst_buffer_map (buffer, &frame->info, GST_MAP_READ); /* FIXME: an atomic remap would be preferable, for now we simply * remap the buffer from RW to RO when using a sysmem allocator, From 67db54d9771d144c79b0cd56ae55c42cebc932d9 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Fri, 25 Sep 2020 22:24:06 +0200 Subject: [PATCH 149/339] vpxdec: allow VideoRegionOfInterestMeta --- .../gst-plugins-good/ext/vpx/gstvpxdec.c | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c index f9a189eef9a..0b6f701e699 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c @@ -96,6 +96,10 @@ gst_vpx_dec_post_processing_flags_get_type (void) #undef C_FLAGS +/* cached quark to avoid contention on the global quark table lock */ +#define META_TAG_VIDEO meta_tag_video_quark +static GQuark meta_tag_video_quark; + static void gst_vpx_dec_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_vpx_dec_get_property (GObject * object, guint prop_id, @@ -111,6 +115,8 @@ gst_vpx_dec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame); static gboolean gst_vpx_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query); +static gboolean gst_vpx_dec_transform_meta (GstVideoDecoder * decoder, + GstVideoCodecFrame * frame, GstMeta * meta); static void gst_vpx_dec_image_to_buffer (GstVPXDec * dec, const vpx_image_t * img, GstBuffer * buffer); @@ -183,6 +189,8 @@ gst_vpx_dec_class_init (GstVPXDecClass * klass) GST_DEBUG_FUNCPTR (gst_vpx_dec_handle_frame);; base_video_decoder_class->decide_allocation = GST_DEBUG_FUNCPTR (gst_vpx_dec_decide_allocation); + base_video_decoder_class->transform_meta = + GST_DEBUG_FUNCPTR (gst_vpx_dec_transform_meta); klass->video_codec_tag = DEFAULT_VIDEO_CODEC_TAG; klass->codec_algo = DEFAULT_CODEC_ALGO; @@ -196,6 +204,8 @@ gst_vpx_dec_class_init (GstVPXDecClass * klass) GST_DEBUG_CATEGORY_INIT (gst_vpxdec_debug, "vpxdec", 0, "VPX Decoder"); + meta_tag_video_quark = g_quark_from_static_string (GST_META_TAG_VIDEO_STR); + gst_type_mark_as_plugin_api (GST_VPX_DEC_TYPE_POST_PROCESSING_FLAGS, 0); gst_type_mark_as_plugin_api (GST_TYPE_VPX_DEC, 0); } @@ -827,6 +837,25 @@ gst_vpx_dec_decide_allocation (GstVideoDecoder * bdec, GstQuery * query) return TRUE; } +static gboolean +gst_vpx_dec_transform_meta (GstVideoDecoder * + decoder, GstVideoCodecFrame * frame, GstMeta * meta) +{ + const GstMetaInfo *roi_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + const GstMetaInfo *info = meta->info; + const gchar *const *tags; + + tags = gst_meta_api_type_get_tags (info->api); + + if (!tags + || info->api == roi_info->api + || (g_strv_length ((gchar **) tags) == 1 + && gst_meta_api_type_has_tag (info->api, META_TAG_VIDEO))) + return TRUE; + + return FALSE; +} + static void gst_vpx_dec_set_stream_info (GstVPXDec * dec, vpx_codec_stream_info_t * stream_info) From 1495c43493219aa8f2b8dd5b557835a38da97bc7 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 25 May 2018 15:36:43 +0200 Subject: [PATCH 150/339] speexenc: don't send silly segments mid-stream [pexhack] --- subprojects/gst-plugins-good/ext/speex/gstspeexenc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c b/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c index 07b0311e4ad..abdd6ecfd9f 100644 --- a/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c +++ b/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c @@ -540,8 +540,8 @@ gst_speex_enc_encode (GstSpeexEnc * enc, GstBuffer * buf) gsize bsize, size; GstBuffer *outbuf; GstFlowReturn ret = GST_FLOW_OK; - GstSegment *segment; - GstClockTime duration; + //GstSegment *segment; + //GstClockTime duration; if (G_LIKELY (buf)) { gst_buffer_map (buf, &map, GST_MAP_READ); @@ -551,6 +551,7 @@ gst_speex_enc_encode (GstSpeexEnc * enc, GstBuffer * buf) if (G_UNLIKELY (bsize % bytes)) { GST_DEBUG_OBJECT (enc, "draining; adding silence samples"); +#if 0 /* If encoding part of a frame, and we have no set stop time on * the output segment, we update the segment stop time to reflect * the last sample. This will let oggmux set the last page's @@ -573,6 +574,7 @@ gst_speex_enc_encode (GstSpeexEnc * enc, GstBuffer * buf) gst_pad_push_event (GST_AUDIO_ENCODER_SRC_PAD (enc), gst_event_new_segment (segment)); } +#endif size = ((bsize / bytes) + 1) * bytes; data0 = data = g_malloc0 (size); From 8aa3d227f27a12702e6b2a2258fadab72a63b601 Mon Sep 17 00:00:00 2001 From: Haakon Sporsheim Date: Mon, 16 Jun 2014 16:34:29 +0200 Subject: [PATCH 151/339] rtph263ppay: remove annex-? fields in caps if not present [pexhack] Change so that if the intersection of upstream and template caps does not contain a specific annex it will be not added to the caps. Previous behavior was that it was set explictily to false. This fixes an issue if an encoder (avenc_h263p) is linked late with the payloader it could end up as not negotiated. --- .../gst-plugins-good/gst/rtp/gstrtph263ppay.c | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph263ppay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph263ppay.c index b6d8040aea6..5eb13ea6bf2 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph263ppay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph263ppay.c @@ -494,7 +494,6 @@ gst_rtp_h263p_pay_sink_getcaps (GstRTPBasePayload * payload, GstPad * pad, caps = gst_caps_merge_structure (caps, new_s); } } else { - gboolean f = FALSE, i = FALSE, j = FALSE, t = FALSE; /* FIXME: ffmpeg support the Appendix K too, how do we express it ? * guint k; */ @@ -503,52 +502,45 @@ gst_rtp_h263p_pay_sink_getcaps (GstRTPBasePayload * payload, GstPad * pad, "variant", G_TYPE_STRING, "itu", NULL); gboolean added = FALSE; + GValue list = { 0 }; + GValue vstr = { 0 }; + + g_value_init (&list, GST_TYPE_LIST); + g_value_init (&vstr, G_TYPE_STRING); + + g_value_set_static_string (&vstr, "h263"); + gst_value_list_append_value (&list, &vstr); + g_value_set_static_string (&vstr, "h263p"); + gst_value_list_append_value (&list, &vstr); + g_value_unset (&vstr); + + gst_structure_set_value (new_s, "h263version", &list); + g_value_unset (&list); str = gst_structure_get_string (s, "f"); - if (str && !strcmp (str, "1")) - f = TRUE; + if (str) { + gst_structure_set (new_s, "annex-f", + G_TYPE_BOOLEAN, !strcmp (str, "1"), NULL); + } str = gst_structure_get_string (s, "i"); - if (str && !strcmp (str, "1")) - i = TRUE; + if (str) { + gst_structure_set (new_s, "annex-i", + G_TYPE_BOOLEAN, !strcmp (str, "1"), NULL); + } str = gst_structure_get_string (s, "j"); - if (str && !strcmp (str, "1")) - j = TRUE; + if (str) { + gst_structure_set (new_s, "annex-j", + G_TYPE_BOOLEAN, !strcmp (str, "1"), NULL); + } str = gst_structure_get_string (s, "t"); - if (str && !strcmp (str, "1")) - t = TRUE; - - if (f || i || j || t) { - GValue list = { 0 }; - GValue vstr = { 0 }; - - g_value_init (&list, GST_TYPE_LIST); - g_value_init (&vstr, G_TYPE_STRING); - - g_value_set_static_string (&vstr, "h263"); - gst_value_list_append_value (&list, &vstr); - g_value_set_static_string (&vstr, "h263p"); - gst_value_list_append_value (&list, &vstr); - g_value_unset (&vstr); - - gst_structure_set_value (new_s, "h263version", &list); - g_value_unset (&list); - } else { - gst_structure_set (new_s, "h263version", G_TYPE_STRING, "h263", NULL); + if (str) { + gst_structure_set (new_s, "annex-t", + G_TYPE_BOOLEAN, !strcmp (str, "1"), NULL); } - if (!f) - gst_structure_set (new_s, "annex-f", G_TYPE_BOOLEAN, FALSE, NULL); - if (!i) - gst_structure_set (new_s, "annex-i", G_TYPE_BOOLEAN, FALSE, NULL); - if (!j) - gst_structure_set (new_s, "annex-j", G_TYPE_BOOLEAN, FALSE, NULL); - if (!t) - gst_structure_set (new_s, "annex-t", G_TYPE_BOOLEAN, FALSE, NULL); - - str = gst_structure_get_string (s, "custom"); if (str) { unsigned int xmax, ymax, mpi; From 483069b8d9531b0211f28d2c93c0a66fd7b4606b Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 28 Jul 2014 11:55:43 +0200 Subject: [PATCH 152/339] rtpg722pay: rtpg722depay: Change payload type to dynamic [pexhack] --- subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c index 060c7e1c8c5..9461b40a7b7 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c @@ -53,7 +53,7 @@ static GstStaticPadTemplate gst_rtp_g722_depay_sink_template = "encoding-name = (string) \"G722\";" "application/x-rtp, " "media = (string) \"audio\", " - "payload = (int) " GST_RTP_PAYLOAD_G722_STRING ", " + "payload = (int) " GST_RTP_PAYLOAD_DYNAMIC_STRING ", " "clock-rate = (int) [ 1, MAX ]" /* "channels = (int) [1, MAX]" */ /* "emphasis = (string) ANY" */ From 535ea129a95e59d5e34712de06ea4822880f99d9 Mon Sep 17 00:00:00 2001 From: Stian Selnes Date: Mon, 7 Jan 2019 10:53:42 +0100 Subject: [PATCH 153/339] rtph263: Fix test to use latest avenc encoder properties --- subprojects/gst-plugins-good/tests/check/elements/rtph263.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtph263.c b/subprojects/gst-plugins-good/tests/check/elements/rtph263.c index 28c01981fe2..afdba783d70 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtph263.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtph263.c @@ -133,8 +133,7 @@ GST_START_TEST (test_h263pay_mode_b_snow) if (!have_element ("avenc_h263")) return; - h = gst_harness_new_parse - ("avenc_h263 rtp-payload-size=1 ! rtph263pay mtu=1350 "); + h = gst_harness_new_parse ("avenc_h263 ps=1 ! rtph263pay mtu=1350 "); gst_harness_add_src_parse (h, "videotestsrc pattern=snow is-live=1 ! " "capsfilter caps=\"video/x-raw,format=I420,width=176,height=144\"", TRUE); From 81aedebc7498e006285d6da2306814f60e0cd2f1 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 2 Oct 2018 16:47:53 +0200 Subject: [PATCH 154/339] multiudpsink: downgrade WARNING to INFO These can spam a lot if it actually happens. --- subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c b/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c index 1f3dca6fced..f13e35b7b31 100644 --- a/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c +++ b/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c @@ -681,14 +681,14 @@ gst_multiudpsink_send_messages (GstMultiUDPSink * sink, GSocket * socket, skip = 1; if (msg_size > UDP_MAX_SIZE) { if (!sent_max_size_warning) { - GST_ELEMENT_WARNING (sink, RESOURCE, WRITE, + GST_ELEMENT_INFO (sink, RESOURCE, WRITE, ("Attempting to send a UDP packets larger than maximum size " "(%u > %d)", msg_size, UDP_MAX_SIZE), ("Reason: %s", err ? err->message : "unknown reason")); sent_max_size_warning = FALSE; } } else { - GST_ELEMENT_WARNING (sink, RESOURCE, WRITE, + GST_ELEMENT_INFO (sink, RESOURCE, WRITE, ("Error sending UDP packets"), ("client %s, reason: %s", gst_udp_address_get_string (msg->address, astr, sizeof (astr)), (err != NULL) ? err->message : "unknown reason")); From 18b5d1f717883d6017ca6b66f6eb985d4fea7761 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 12 Apr 2018 12:52:44 +0200 Subject: [PATCH 155/339] qtdemux: initialize variable to stop warning --- subprojects/gst-plugins-good/gst/isomp4/qtdemux.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c index 3923456d086..5ceaa30bbbf 100644 --- a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c +++ b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c @@ -11699,7 +11699,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak) GNode *stbl; GNode *stsd; GNode *mp4a; - GNode *mp4v; + GNode *mp4v = NULL; GNode *esds; GNode *tref; GNode *udta; From 9882be2ad6bf720e777215e130c4b21e85d71768 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Wed, 9 Jan 2019 13:29:26 +0100 Subject: [PATCH 156/339] qtdemux: fix leak in Opus caps The caps for Opus is generated via a utility function that creates a new GstCaps instance after validating the codec arguments. There are possible ways to solve this, we chose the one requiring the least number of modifications. It would be also possible to simply set the caps with the specified arguments; however, this will required to move all the validation to qtdemux or require a change to gstreamer codec-utils. --- subprojects/gst-plugins-good/gst/isomp4/qtdemux.c | 1 + 1 file changed, 1 insertion(+) diff --git a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c index 5ceaa30bbbf..40834aabb36 100644 --- a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c +++ b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c @@ -13582,6 +13582,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak) goto corrupt_file; } + gst_caps_unref (entry->caps); entry->caps = gst_codec_utils_opus_create_caps (rate, n_channels, channel_mapping_family, stream_count, coupled_count, channel_mapping); From 3e77291e9afc735c3bef0b5b4dea9fdc00a8f463 Mon Sep 17 00:00:00 2001 From: Richard Harrison Date: Wed, 9 Oct 2019 22:15:07 +0100 Subject: [PATCH 157/339] qtdemux: Expose max raw audio samples per buffer as a property, revert to upstream default of 4096. --- .../gst-plugins-good/gst/isomp4/qtdemux.c | 53 ++++++++++++++++++- .../gst-plugins-good/gst/isomp4/qtdemux.h | 4 ++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c index 40834aabb36..02b55836612 100644 --- a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c +++ b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c @@ -115,6 +115,12 @@ GST_DEBUG_CATEGORY (qtdemux_debug); typedef struct _QtDemuxCencSampleSetInfo QtDemuxCencSampleSetInfo; typedef struct _QtDemuxAavdEncryptionInfo QtDemuxAavdEncryptionInfo; +enum +{ + PROP_0, + PROP_MAX_AUDIO_SAMPLES +}; + /* Macros for converting to/from timescale */ #define QTSTREAMTIME_TO_GSTTIME(stream, value) (gst_util_uint64_scale((value), GST_SECOND, (stream)->timescale)) #define GSTTIME_TO_QTSTREAMTIME(stream, value) (gst_util_uint64_scale((value), (stream)->timescale, GST_SECOND)) @@ -402,6 +408,42 @@ static void gst_qtdemux_reset (GstQTDemux * qtdemux, gboolean hard); static void qtdemux_clear_protection_events_on_all_streams (GstQTDemux * qtdemux); +static void +gst_qtdemux_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQTDemux *qtdemux = GST_QTDEMUX (object); + + GST_OBJECT_LOCK (qtdemux); + switch (prop_id) { + case PROP_MAX_AUDIO_SAMPLES: + qtdemux->max_audio_samples = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_UNLOCK (qtdemux); +} + +static void +gst_qtdemux_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstQTDemux *qtdemux = GST_QTDEMUX (object); + + GST_OBJECT_LOCK (qtdemux); + switch (prop_id) { + case PROP_MAX_AUDIO_SAMPLES: + g_value_set_uint (value, qtdemux->max_audio_samples); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_UNLOCK (qtdemux); +} + static void gst_qtdemux_class_init (GstQTDemuxClass * klass) { @@ -416,6 +458,9 @@ gst_qtdemux_class_init (GstQTDemuxClass * klass) gobject_class->dispose = gst_qtdemux_dispose; gobject_class->finalize = gst_qtdemux_finalize; + gobject_class->get_property = gst_qtdemux_get_property; + gobject_class->set_property = gst_qtdemux_set_property; + gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_qtdemux_change_state); #if 0 gstelement_class->set_index = GST_DEBUG_FUNCPTR (gst_qtdemux_set_index); @@ -438,6 +483,11 @@ gst_qtdemux_class_init (GstQTDemuxClass * klass) "Demultiplex a QuickTime file into audio and video streams", "David Schleef , Wim Taymans "); + g_object_class_install_property (gobject_class, PROP_MAX_AUDIO_SAMPLES, + g_param_spec_uint ("max-audio-samples", "Max audio samples", + "Maximum raw audio samples per buffer", 1, G_MAXUINT, 4096, + G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + GST_DEBUG_CATEGORY_INIT (qtdemux_debug, "qtdemux", 0, "qtdemux plugin"); gst_riff_init (); } @@ -16182,7 +16232,8 @@ qtdemux_audio_caps (GstQTDemux * qtdemux, QtDemuxStream * stream, if (g_str_has_prefix (name, "audio/x-raw")) { stream->need_clip = TRUE; stream->min_buffer_size = 1024 * entry->bytes_per_frame; - stream->max_buffer_size = 4096 * entry->bytes_per_frame; + stream->max_buffer_size = + qtdemux->max_audio_samples * entry->bytes_per_frame; GST_DEBUG ("setting min/max buffer sizes to %d/%d", stream->min_buffer_size, stream->max_buffer_size); } diff --git a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.h b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.h index 6e7f64b91c5..c8922349807 100644 --- a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.h +++ b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.h @@ -309,6 +309,10 @@ struct _GstQTDemux { * fields. */ gboolean received_seek; gboolean first_moof_already_parsed; + + /** Maximum number of audio samples per buffer when demuxing raw audio. + * Used to determine max buffer size for raw audio streams. */ + guint max_audio_samples; }; struct _GstQTDemuxClass { From d82759061a60d4c9c7748e9fb81c8eff3b491195 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 11 Aug 2021 12:52:40 +0200 Subject: [PATCH 158/339] vp9enc: if VP9E_SET_ROW_MT is not available, don't use it --- subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c | 4 ++++ subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c index a9a94626be6..3ee716ed4b1 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c @@ -314,6 +314,7 @@ gst_vp9_enc_set_property (GObject * object, guint prop_id, case PROP_ROW_MT: gst_vp9_enc->row_mt = g_value_get_boolean (value); if (gst_vpx_enc->inited) { +#ifdef VPX_CTRL_VP9E_SET_ROW_MT status = vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_ROW_MT, gst_vp9_enc->row_mt ? 1 : 0); @@ -321,6 +322,7 @@ gst_vp9_enc_set_property (GObject * object, guint prop_id, GST_WARNING_OBJECT (gst_vpx_enc, "Failed to set VP9E_SET_ROW_MT: %s", gst_vpx_error_name (status)); } +#endif } break; case PROP_AQ_MODE: @@ -496,6 +498,7 @@ gst_vp9_enc_configure_encoder (GstVPXEnc * encoder, GstVideoCodecState * state) GST_DEBUG_OBJECT (encoder, "Failed to set VP9E_SET_TILE_ROWS: %s", gst_vpx_error_name (status)); } +#ifdef VPX_CTRL_VP9E_SET_ROW_MT status = vpx_codec_control (&encoder->encoder, VP9E_SET_ROW_MT, vp9enc->row_mt ? 1 : 0); @@ -503,6 +506,7 @@ gst_vp9_enc_configure_encoder (GstVPXEnc * encoder, GstVideoCodecState * state) GST_DEBUG_OBJECT (encoder, "Failed to set VP9E_SET_ROW_MT: %s", gst_vpx_error_name (status)); } +#endif status = vpx_codec_control (&encoder->encoder, VP9E_SET_AQ_MODE, vp9enc->aq_mode); if (status != VPX_CODEC_OK) { diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h index 03d31d1bd23..45540d050e8 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h @@ -47,9 +47,7 @@ struct _GstVP9Enc guint tile_columns; guint tile_rows; -#ifdef VPX_CTRL_VP9E_SET_ROW_MT gboolean row_mt; -#endif GstVPXAQ aq_mode; gboolean frame_parallel_decoding; }; From eacebb7b49d05f460a359b972ab7ebf5607e6cb6 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 23 Aug 2021 16:12:30 +0200 Subject: [PATCH 159/339] rtpvp8depay: pexip changes --- .../gst-plugins-good/gst/rtp/gstrtpvp8depay.c | 760 ++++++++++-------- .../gst-plugins-good/gst/rtp/gstrtpvp8depay.h | 13 +- 2 files changed, 446 insertions(+), 327 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.c index 708bceb688c..2bf9eb46f22 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.c @@ -34,10 +34,6 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_vp8_depay_debug); #define GST_CAT_DEFAULT gst_rtp_vp8_depay_debug static void gst_rtp_vp8_depay_dispose (GObject * object); -static void gst_rtp_vp8_depay_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec); -static void gst_rtp_vp8_depay_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec); static GstBuffer *gst_rtp_vp8_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp); static GstStateChangeReturn gst_rtp_vp8_depay_change_state (GstElement * @@ -66,29 +62,216 @@ GST_STATIC_PAD_TEMPLATE ("sink", "media = (string) \"video\"," "encoding-name = (string) { \"VP8\", \"VP8-DRAFT-IETF-01\" }")); -#define DEFAULT_WAIT_FOR_KEYFRAME FALSE -#define DEFAULT_REQUEST_KEYFRAME FALSE - enum { PROP_0, PROP_WAIT_FOR_KEYFRAME, - PROP_REQUEST_KEYFRAME, + PROP_HIDE_PICTURE_ID_GAP, }; +typedef struct _GstVP8PacketInfo +{ + guint size; + gboolean frame_start; + gboolean end_of_frame; + guint8 part_start; + guint8 is_non_ref_frame; + gboolean temporally_scaled; + guint8 layer_sync; + guint8 part_idx; + guint32 picture_id; + guint8 tl0picidx; + guint8 tid; + guint8 temporal_key_idx; + guint8 hdrsize; + GstClockTime pts; +} GstVP8PacketInfo; + #define PICTURE_ID_NONE (UINT_MAX) #define IS_PICTURE_ID_15BITS(pid) (((guint)(pid) & 0x8000) != 0) +#define DEFAULT_WAIT_FOR_KEYFRAME FALSE +#define DEFAULT_HIDE_PICTURE_ID_GAP FALSE + +// VP8 Payload Descriptor Format +// (see RFC:7741 Section-4.2) +// 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// |X|R|N|S|R| PID | (REQUIRED) |X|R|N|S|R| PID | (REQUIRED) +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// X: |I|L|T|K| RSV | (OPTIONAL) X: |I|L|T|K| RSV | (OPTIONAL) +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// I: |M| PictureID | (OPTIONAL) I: |M| PictureID | (OPTIONAL) +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// L: | TL0PICIDX | (OPTIONAL) | PictureID | +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// T/K: |TID|Y| KEYIDX | (OPTIONAL) L: | TL0PICIDX | (OPTIONAL) +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// T/K: |TID|Y| KEYIDX | (OPTIONAL) +// +-+-+-+-+-+-+-+-+ +static gboolean +gst_rtp_vp8_depay_parse_header (GstRTPBuffer * rtp, GstVP8PacketInfo * out) +{ + guint8 has_ext_ctrl_bits = FALSE; + guint8 *data = gst_rtp_buffer_get_payload (rtp); + guint size = gst_rtp_buffer_get_payload_len (rtp); + GstBitReader br = GST_BIT_READER_INIT (data, size); + + /* At least one header and one vp8 byte */ + if (G_UNLIKELY (size < 2)) return FALSE; + + out->size = size; + out->pts = GST_BUFFER_PTS (rtp->buffer); + out->end_of_frame = gst_rtp_buffer_get_marker (rtp); + out->picture_id = PICTURE_ID_NONE; + out->temporally_scaled = FALSE; + out->tl0picidx = 0; + out->tid = 0; + out->layer_sync = 0; + out->temporal_key_idx = 0; + out->hdrsize = 0; + +#define fail_if(val) if (G_UNLIKELY (val)) return FALSE; + /* Extended control bits present (X bit) */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &has_ext_ctrl_bits, 1)); + /* Reserved bit (R bit) */ + fail_if (!gst_bit_reader_skip (&br, 1)); + /* Non-reference frame (N bit) */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->is_non_ref_frame, 1)); + /* Start of VP8 partition (S bit) */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->part_start, 1)); + /* Reserved bit (R bit) */ + fail_if (!gst_bit_reader_skip (&br, 1)); + /* Partition index (PID bits) */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->part_idx, 3)); + + out->frame_start = (out->part_start && !out->part_idx); + + /* Check X optional header */ + if (has_ext_ctrl_bits) { + guint8 has_picture_id = 0; + guint8 has_tl0picidx = 0; + guint8 tid_set = 0; + guint8 keyidx_set = 0; + + /* Check I optional header */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &has_picture_id, 1)); + /* Check L optional header */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &has_tl0picidx, 1)); + /* Check T is set */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &tid_set, 1)); + /* Check K is set */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &keyidx_set, 1)); + /* Reserved bit (R bit) */ + fail_if (!gst_bit_reader_skip (&br, 4)); + + /* Stream is temporally scaled if L or T bits are set */ + out->temporally_scaled = (has_tl0picidx || tid_set); + + if (has_picture_id) { + guint8 is_ext_pic_id = 0; + fail_if (!gst_bit_reader_peek_bits_uint8 (&br, &is_ext_pic_id, 1)); + fail_if (!gst_bit_reader_get_bits_uint32 (&br, &out->picture_id, is_ext_pic_id ? 16 : 8)); + } + + if (has_tl0picidx) { + /* TL0PICIDX must be ignored unless T is set */ + if (tid_set) { + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->tl0picidx, 8)); + } else { + fail_if (!gst_bit_reader_skip (&br, 8)); + } + } + + if (tid_set || keyidx_set) { + /* TID and Y must be ignored unless T is set */ + if (tid_set) { + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->tid, 2)); + fail_if (!gst_bit_reader_peek_bits_uint8 (&br, &out->layer_sync, 1)); + } else { + fail_if (!gst_bit_reader_skip (&br, 3)); + } + /* KEYIDX must be ignored unless K is set */ + if (keyidx_set) { + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->temporal_key_idx, 5)); + } else { + fail_if (!gst_bit_reader_skip (&br, 5)); + } + } + } +#undef fail_if + out->hdrsize = (guint8) ((gst_bit_reader_get_pos (&br) + 1) / 8); + return TRUE; +} + +typedef struct _GstVP8PFrameInfo +{ + gboolean is_keyframe; + guint profile; + guint width; + guint height; +} GstVP8PFrameInfo; + +static gboolean +gst_rtp_vp8_depay_parse_frame_descriptor (GstRtpVP8Depay * self, + GstVP8PFrameInfo * out) +{ + guint8 header[10]; + + if (gst_adapter_available (self->adapter) < 10) { + return FALSE; + } + + gst_adapter_copy (self->adapter, &header, 0, 10); + out->is_keyframe = !(header[0] & 0x01); + out->profile = (header[0] & 0x0e) >> 1; + out->width = GST_READ_UINT16_LE (header + 6) & 0x3fff; + out->height = GST_READ_UINT16_LE (header + 8) & 0x3fff; + return TRUE; +} + static void -gst_rtp_vp8_depay_init (GstRtpVP8Depay * self) +gst_rtp_vp8_depay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) { - gst_rtp_base_depayload_set_aggregate_hdrext_enabled (GST_RTP_BASE_DEPAYLOAD - (self), TRUE); + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (object); + switch (prop_id) { + case PROP_WAIT_FOR_KEYFRAME: + self->wait_for_keyframe = g_value_get_boolean (value); + break; + case PROP_HIDE_PICTURE_ID_GAP: + self->hide_picture_id_gap = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} +static void +gst_rtp_vp8_depay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (object); + switch (prop_id) { + case PROP_WAIT_FOR_KEYFRAME: + g_value_set_boolean (value, self->wait_for_keyframe); + break; + case PROP_HIDE_PICTURE_ID_GAP: + g_value_set_boolean (value, self->hide_picture_id_gap); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_rtp_vp8_depay_init (GstRtpVP8Depay * self) +{ self->adapter = gst_adapter_new (); self->started = FALSE; self->wait_for_keyframe = DEFAULT_WAIT_FOR_KEYFRAME; - self->request_keyframe = DEFAULT_REQUEST_KEYFRAME; self->last_pushed_was_lost_event = FALSE; } @@ -114,30 +297,23 @@ gst_rtp_vp8_depay_class_init (GstRtpVP8DepayClass * gst_rtp_vp8_depay_class) object_class->set_property = gst_rtp_vp8_depay_set_property; object_class->get_property = gst_rtp_vp8_depay_get_property; + element_class->change_state = gst_rtp_vp8_depay_change_state; + + depay_class->process_rtp_packet = gst_rtp_vp8_depay_process; + depay_class->handle_event = gst_rtp_vp8_depay_handle_event; + depay_class->packet_lost = gst_rtp_vp8_depay_packet_lost; + g_object_class_install_property (object_class, PROP_WAIT_FOR_KEYFRAME, g_param_spec_boolean ("wait-for-keyframe", "Wait for Keyframe", "Wait for the next keyframe after packet loss", DEFAULT_WAIT_FOR_KEYFRAME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - /** - * GstRtpVP8Depay:request-keyframe: - * - * Request new keyframe when packet loss is detected - * - * Since: 1.20 - */ - g_object_class_install_property (object_class, PROP_REQUEST_KEYFRAME, - g_param_spec_boolean ("request-keyframe", "Request Keyframe", - "Request new keyframe when packet loss is detected", - DEFAULT_REQUEST_KEYFRAME, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - element_class->change_state = gst_rtp_vp8_depay_change_state; - - depay_class->process_rtp_packet = gst_rtp_vp8_depay_process; - depay_class->handle_event = gst_rtp_vp8_depay_handle_event; - depay_class->packet_lost = gst_rtp_vp8_depay_packet_lost; + g_object_class_install_property (object_class, PROP_HIDE_PICTURE_ID_GAP, + g_param_spec_boolean ("hide-picture-id-gap", "Hide Picture ID Gap", + "Wether to trigger a key-unit request when there is a gap in " + "the picture ID", DEFAULT_HIDE_PICTURE_ID_GAP, + G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); GST_DEBUG_CATEGORY_INIT (gst_rtp_vp8_depay_debug, "rtpvp8depay", 0, "VP8 Video RTP Depayloader"); @@ -158,64 +334,27 @@ gst_rtp_vp8_depay_dispose (GObject * object) G_OBJECT_CLASS (gst_rtp_vp8_depay_parent_class)->dispose (object); } -static void -gst_rtp_vp8_depay_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) -{ - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (object); - - switch (prop_id) { - case PROP_WAIT_FOR_KEYFRAME: - self->wait_for_keyframe = g_value_get_boolean (value); - break; - case PROP_REQUEST_KEYFRAME: - self->request_keyframe = g_value_get_boolean (value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static void -gst_rtp_vp8_depay_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (object); - - switch (prop_id) { - case PROP_WAIT_FOR_KEYFRAME: - g_value_set_boolean (value, self->wait_for_keyframe); - break; - case PROP_REQUEST_KEYFRAME: - g_value_set_boolean (value, self->request_keyframe); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - static gint picture_id_compare (guint16 id0, guint16 id1) { guint shift = 16 - (IS_PICTURE_ID_15BITS (id1) ? 15 : 7); - id0 = id0 << shift; - id1 = id1 << shift; + id0 = (guint16) (id0 << shift); + id1 = (guint16) (id1 << shift); return ((gint16) (id1 - id0)) >> shift; } static void -send_last_lost_event (GstRtpVP8Depay * self) +send_last_lost_event (GstRtpVP8Depay * self, const gchar * reason) { if (self->last_lost_event) { - GST_ERROR_OBJECT (self, - "Sending the last stopped lost event: %" GST_PTR_FORMAT, - self->last_lost_event); + GST_DEBUG_OBJECT (self, + "Sending the last stopped lost event: %" GST_PTR_FORMAT + " reason \"%s\"", self->last_lost_event, reason ? reason : "None"); GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp8_depay_parent_class) ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), self->last_lost_event); - gst_event_replace (&self->last_lost_event, NULL); + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; self->last_pushed_was_lost_event = TRUE; } } @@ -234,7 +373,8 @@ send_new_lost_event (GstRtpVP8Depay * self, GstClockTime timestamp, event = gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, gst_structure_new ("GstRTPPacketLost", "timestamp", G_TYPE_UINT64, timestamp, - "duration", G_TYPE_UINT64, G_GUINT64_CONSTANT (0), NULL)); + "duration", G_TYPE_UINT64, G_GUINT64_CONSTANT (0), + "no-packet-loss", G_TYPE_BOOLEAN, self->hide_picture_id_gap, NULL)); GST_DEBUG_OBJECT (self, "Pushing lost event " "(picids 0x%x 0x%x, reason \"%s\"): %" GST_PTR_FORMAT, @@ -244,257 +384,262 @@ send_new_lost_event (GstRtpVP8Depay * self, GstClockTime timestamp, ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), event); gst_event_unref (event); + self->last_pushed_was_lost_event = TRUE; +} + +static void +send_lost_event (GstRtpVP8Depay * self, GstClockTime timestamp, + guint picture_id, const gchar * reason) +{ + if (self->last_lost_event) { + send_last_lost_event (self, reason); + } else { + /* FIXME: Add property to control whether to send GAP events */ + send_new_lost_event (self, timestamp, picture_id, reason); + } +} + + +static void +send_keyframe_request (GstRtpVP8Depay * self) +{ + GST_DEBUG_OBJECT (self, "Sending keyframe request"); + if (!gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (self), + gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, + TRUE, 0))) { + GST_ERROR_OBJECT (self, "Failed to push keyframe request"); + } +} + + +static void +drop_last_lost_event (GstRtpVP8Depay * self) +{ + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } } static void -send_last_lost_event_if_needed (GstRtpVP8Depay * self, guint new_picture_id) +gst_rtp_vp8_depay_hadle_picture_id_gap (GstRtpVP8Depay * self, + guint new_picture_id, GstClockTime lost_event_timestamp) { + const gchar *reason = NULL; + gboolean fwd_last_lost_event = FALSE; + gboolean create_lost_event = FALSE; + gboolean gap_event_sent = FALSE; + if (self->last_picture_id == PICTURE_ID_NONE) return; + if (new_picture_id == PICTURE_ID_NONE) { + reason = "picture id does not exist"; + fwd_last_lost_event = TRUE; + } else if (IS_PICTURE_ID_15BITS (self->last_picture_id) && + !IS_PICTURE_ID_15BITS (new_picture_id)) { + reason = "picture id has less bits than before"; + fwd_last_lost_event = TRUE; + } else if (picture_id_compare ((guint16) self->last_picture_id, + (guint16) new_picture_id) != 1) { + reason = "picture id gap"; + fwd_last_lost_event = TRUE; + /* Only create a new one if we just didn't push a lost event */ + create_lost_event = self->last_pushed_was_lost_event == FALSE; + } + if (self->last_lost_event) { - gboolean send_lost_event = FALSE; - if (new_picture_id == PICTURE_ID_NONE) { - GST_DEBUG_OBJECT (self, "Dropping the last stopped lost event " - "(picture id does not exist): %" GST_PTR_FORMAT, - self->last_lost_event); - } else if (IS_PICTURE_ID_15BITS (self->last_picture_id) && - !IS_PICTURE_ID_15BITS (new_picture_id)) { - GST_DEBUG_OBJECT (self, "Dropping the last stopped lost event " - "(picture id has less bits than before): %" GST_PTR_FORMAT, - self->last_lost_event); - } else if (picture_id_compare (self->last_picture_id, new_picture_id) != 1) { - GstStructure *s = gst_event_writable_structure (self->last_lost_event); - - GST_DEBUG_OBJECT (self, "Sending the last stopped lost event " - "(gap in picture id %u %u): %" GST_PTR_FORMAT, - self->last_picture_id, new_picture_id, self->last_lost_event); - send_lost_event = TRUE; - /* Prevent rtpbasedepayload from dropping the event now - * that we have made sure the lost packet was not FEC */ - gst_structure_remove_field (s, "might-have-been-fec"); + if (fwd_last_lost_event) { + send_last_lost_event (self, reason); + gap_event_sent = TRUE; + } else { + drop_last_lost_event (self); } - if (send_lost_event) - GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp8_depay_parent_class) - ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), - self->last_lost_event); + } + + if (create_lost_event && !gap_event_sent) { + send_new_lost_event (self, lost_event_timestamp, new_picture_id, reason); + gap_event_sent = TRUE; + } - gst_event_replace (&self->last_lost_event, NULL); + if (gap_event_sent && self->waiting_for_keyframe) { + send_keyframe_request (self); } } -static GstBuffer * -gst_rtp_vp8_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) + +static gboolean +gst_rtp_vp8_depay_reset_current_frame (GstRtpVP8Depay * self, + GstVP8PacketInfo * packet_info, const gchar * reason) { - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (depay); - GstBuffer *payload; - guint8 *data; - guint hdrsize = 1; - guint picture_id = PICTURE_ID_NONE; - guint size = gst_rtp_buffer_get_payload_len (rtp); - guint s_bit; - guint part_id; - gboolean frame_start; - gboolean sent_lost_event = FALSE; + self->started = FALSE; - if (G_UNLIKELY (GST_BUFFER_IS_DISCONT (rtp->buffer))) { - GST_DEBUG_OBJECT (self, "Discontinuity, flushing adapter"); - gst_adapter_clear (self->adapter); - self->started = FALSE; + if (!gst_adapter_available (self->adapter)) return FALSE; - if (self->wait_for_keyframe) - self->waiting_for_keyframe = TRUE; + GST_DEBUG_OBJECT (self, "%s, flushing adapter", reason); + gst_adapter_clear (self->adapter); - if (self->request_keyframe) - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); + // Preventing for flooding with gap_events + if (!self->last_pushed_was_lost_event) { + send_lost_event (self, packet_info->pts, packet_info->picture_id, reason); } - /* At least one header and one vp8 byte */ - if (G_UNLIKELY (size < 2)) - goto too_small; - - data = gst_rtp_buffer_get_payload (rtp); - - s_bit = (data[0] >> 4) & 0x1; - part_id = (data[0] >> 0) & 0x7; + if (self->wait_for_keyframe) { + self->waiting_for_keyframe = TRUE; + } + if (self->waiting_for_keyframe) { + send_keyframe_request (self); + } + return TRUE; +} - /* Check X optional header */ - if ((data[0] & 0x80) != 0) { - hdrsize++; - /* Check I optional header */ - if ((data[1] & 0x80) != 0) { - if (G_UNLIKELY (size < 3)) - goto too_small; - hdrsize++; - /* Check for 16 bits PictureID */ - picture_id = data[2]; - if ((data[2] & 0x80) != 0) { - if (G_UNLIKELY (size < 4)) - goto too_small; - hdrsize++; - picture_id = (picture_id << 8) | data[3]; - } +static GstBuffer * +gst_rtp_vp8_depay_get_frame (GstRtpVP8Depay * self, + const GstVP8PacketInfo * packet_info, const GstVP8PFrameInfo * frame_info) +{ + /* mark keyframes */ + GstBuffer *out = gst_adapter_take_buffer (self->adapter, + gst_adapter_available (self->adapter)); + + out = gst_buffer_make_writable (out); + + /* Filter away all metas that are not sensible to copy */ + gst_rtp_drop_non_video_meta (self, out); + + GstCustomMeta *meta = gst_buffer_add_custom_meta (out, "GstVP8Meta"); + GstStructure *s = gst_custom_meta_get_structure (meta); + gst_structure_set (s, + "use-temporal-scaling", G_TYPE_BOOLEAN, packet_info->temporally_scaled, + "layer-sync", G_TYPE_BOOLEAN, packet_info->layer_sync, + "layer-id", G_TYPE_UINT, packet_info->tid, + "tl0picidx", G_TYPE_UINT, packet_info->tl0picidx, NULL); + + if (frame_info->is_keyframe) { + GST_BUFFER_FLAG_UNSET (out, GST_BUFFER_FLAG_DELTA_UNIT); + GST_DEBUG_OBJECT (self, "Processed keyframe"); + + if (G_UNLIKELY (self->last_width != frame_info->width || + self->last_height != frame_info->height || + self->last_profile != frame_info->profile)) { + gchar profile_str[3]; + GstCaps *srccaps; + + snprintf (profile_str, 3, "%u", frame_info->profile); + srccaps = gst_caps_new_simple ("video/x-vp8", + "framerate", GST_TYPE_FRACTION, 0, 1, + "height", G_TYPE_INT, frame_info->height, + "width", G_TYPE_INT, frame_info->width, + "profile", G_TYPE_STRING, profile_str, NULL); + + gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (self), srccaps); + gst_caps_unref (srccaps); + + self->last_width = frame_info->width; + self->last_height = frame_info->height; + self->last_profile = frame_info->profile; } - /* Check L optional header */ - if ((data[1] & 0x40) != 0) - hdrsize++; - /* Check T or K optional headers */ - if ((data[1] & 0x20) != 0 || (data[1] & 0x10) != 0) - hdrsize++; - } - GST_LOG_OBJECT (depay, - "hdrsize %u, size %u, picture id 0x%x, s %u, part_id %u", hdrsize, size, - picture_id, s_bit, part_id); - if (G_UNLIKELY (hdrsize >= size)) - goto too_small; - - frame_start = (s_bit == 1) && (part_id == 0); - if (frame_start) { - if (G_UNLIKELY (self->started)) { - GST_DEBUG_OBJECT (depay, "Incomplete frame, flushing adapter"); - /* keep the current buffer because it may still be used later */ - gst_rtp_base_depayload_flush (depay, TRUE); - gst_adapter_clear (self->adapter); - self->started = FALSE; - - if (self->wait_for_keyframe) - self->waiting_for_keyframe = TRUE; - if (self->request_keyframe) - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); - - send_new_lost_event (self, GST_BUFFER_PTS (rtp->buffer), picture_id, - "Incomplete frame detected"); - sent_lost_event = TRUE; + self->waiting_for_keyframe = FALSE; + } else { + GST_BUFFER_FLAG_SET (out, GST_BUFFER_FLAG_DELTA_UNIT); + GST_DEBUG_OBJECT (self, "Processed interframe"); + + if (self->waiting_for_keyframe) { + gst_buffer_unref (out); + out = NULL; + GST_INFO_OBJECT (self, "Dropping inter-frame before intra-frame"); + send_keyframe_request (self); } } - if (!self->started) { - if (G_UNLIKELY (!frame_start)) { - GST_DEBUG_OBJECT (depay, - "The frame is missing the first packet, ignoring the packet"); - if (self->stop_lost_events && !sent_lost_event) { - send_last_lost_event (self); - self->stop_lost_events = FALSE; - } + return out; +} - if (self->wait_for_keyframe) - self->waiting_for_keyframe = TRUE; - if (self->request_keyframe) - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); +static GstBuffer * +gst_rtp_vp8_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) +{ + GstBuffer *out = NULL; + GstVP8PacketInfo packet_info; + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (depay); + + if (G_UNLIKELY (!gst_rtp_vp8_depay_parse_header (rtp, &packet_info))) { + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "Invalid rtp packet detected"); + return NULL; + } - goto done; - } + GST_LOG_OBJECT (depay, + "hdrsize %u, size %u, picture id 0x%x, s %u, part_id %u", + packet_info.hdrsize, packet_info.size, packet_info.picture_id, + packet_info.part_start, packet_info.part_idx); + + if (G_UNLIKELY (GST_BUFFER_IS_DISCONT (rtp->buffer))) { + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "Discontinuity detected"); + } + + if (G_UNLIKELY (packet_info.frame_start == self->started)) { + // We either didn't completed previous frame + // or didn't start next frame + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "Incomplete frame detected"); + } + if (packet_info.frame_start) { GST_LOG_OBJECT (depay, "Found the start of the frame"); - if (self->stop_lost_events && !sent_lost_event) { - send_last_lost_event_if_needed (self, picture_id); - self->stop_lost_events = FALSE; - } + gst_rtp_vp8_depay_hadle_picture_id_gap (self, packet_info.picture_id, + packet_info.pts); self->started = TRUE; + self->stop_lost_events = FALSE; + self->last_picture_id = packet_info.picture_id; + + } else if (self->started) { + // PictureID gap in a middle of the frame + if (self->last_picture_id != packet_info.picture_id) + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "picture id gap"); } - payload = gst_rtp_buffer_get_payload_subbuffer (rtp, hdrsize, -1); - gst_adapter_push (self->adapter, payload); - self->last_picture_id = picture_id; - - /* Marker indicates that it was the last rtp packet for this frame */ - if (gst_rtp_buffer_get_marker (rtp)) { - GstBuffer *out; - guint8 header[10]; - - GST_LOG_OBJECT (depay, - "Found the end of the frame (%" G_GSIZE_FORMAT " bytes)", - gst_adapter_available (self->adapter)); - if (gst_adapter_available (self->adapter) < 10) - goto too_small; - gst_adapter_copy (self->adapter, &header, 0, 10); - - out = gst_adapter_take_buffer (self->adapter, - gst_adapter_available (self->adapter)); - - self->started = FALSE; - - /* mark keyframes */ - out = gst_buffer_make_writable (out); - /* Filter away all metas that are not sensible to copy */ - gst_rtp_drop_non_video_meta (self, out); - if ((header[0] & 0x01)) { - GST_BUFFER_FLAG_SET (out, GST_BUFFER_FLAG_DELTA_UNIT); - - if (self->waiting_for_keyframe) { - gst_rtp_base_depayload_flush (depay, FALSE); - gst_buffer_unref (out); - out = NULL; - GST_INFO_OBJECT (self, "Dropping inter-frame before intra-frame"); - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); - } - } else { - guint profile, width, height; + if (self->started) { + /* Store rtp payload data in adapter */ + gst_adapter_push (self->adapter, gst_rtp_buffer_get_payload_subbuffer (rtp, + packet_info.hdrsize, -1)); - GST_BUFFER_FLAG_UNSET (out, GST_BUFFER_FLAG_DELTA_UNIT); - GST_DEBUG_OBJECT (self, "Processed keyframe"); + /* Marker indicates that it was the last rtp packet for this frame */ + if (packet_info.end_of_frame) { + GstVP8PFrameInfo frame_info; - profile = (header[0] & 0x0e) >> 1; - width = GST_READ_UINT16_LE (header + 6) & 0x3fff; - height = GST_READ_UINT16_LE (header + 8) & 0x3fff; + GST_LOG_OBJECT (depay, + "Found the end of the frame (%" G_GSIZE_FORMAT " bytes)", + gst_adapter_available (self->adapter)); - if (G_UNLIKELY (self->last_width != width || - self->last_height != height || self->last_profile != profile)) { - gchar profile_str[3]; - GstCaps *srccaps; + if (gst_rtp_vp8_depay_parse_frame_descriptor (self, &frame_info)) { + out = gst_rtp_vp8_depay_get_frame (self, &packet_info, &frame_info); - snprintf (profile_str, 3, "%u", profile); - srccaps = gst_caps_new_simple ("video/x-vp8", - "framerate", GST_TYPE_FRACTION, 0, 1, - "height", G_TYPE_INT, height, - "width", G_TYPE_INT, width, - "profile", G_TYPE_STRING, profile_str, NULL); + if (packet_info.picture_id != PICTURE_ID_NONE) + self->stop_lost_events = TRUE; - gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (depay), srccaps); - gst_caps_unref (srccaps); - - self->last_width = width; - self->last_height = height; - self->last_profile = profile; + self->last_pushed_was_lost_event = FALSE; + self->started = FALSE; + } else { + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "Invalid rtp packet detected"); } - self->waiting_for_keyframe = FALSE; } - - if (picture_id != PICTURE_ID_NONE) - self->stop_lost_events = TRUE; - - self->last_pushed_was_lost_event = FALSE; - - return out; + } else { + // Wating for start of the new frame + GST_DEBUG_OBJECT (depay, + "The frame is missing the first packet, ignoring the packet"); } -done: - gst_rtp_base_depayload_dropped (depay); - return NULL; - -too_small: - GST_DEBUG_OBJECT (self, "Invalid rtp packet (too small), ignoring"); - gst_rtp_base_depayload_flush (depay, FALSE); - gst_adapter_clear (self->adapter); - self->started = FALSE; - - goto done; + return out; } static GstStateChangeReturn gst_rtp_vp8_depay_change_state (GstElement * element, GstStateChange transition) { - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (element); + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (element); switch (transition) { case GST_STATE_CHANGE_READY_TO_PAUSED: @@ -504,7 +649,10 @@ gst_rtp_vp8_depay_change_state (GstElement * element, GstStateChange transition) self->waiting_for_keyframe = TRUE; self->caps_sent = FALSE; self->last_picture_id = PICTURE_ID_NONE; - gst_event_replace (&self->last_lost_event, NULL); + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } self->stop_lost_events = FALSE; break; default: @@ -519,7 +667,7 @@ gst_rtp_vp8_depay_change_state (GstElement * element, GstStateChange transition) static gboolean gst_rtp_vp8_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) { - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (depay); + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (depay); switch (GST_EVENT_TYPE (event)) { case GST_EVENT_FLUSH_STOP: @@ -527,7 +675,10 @@ gst_rtp_vp8_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) self->last_height = -1; self->last_width = -1; self->last_picture_id = PICTURE_ID_NONE; - gst_event_replace (&self->last_lost_event, NULL); + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } self->stop_lost_events = FALSE; break; default: @@ -542,45 +693,18 @@ gst_rtp_vp8_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) static gboolean gst_rtp_vp8_depay_packet_lost (GstRTPBaseDepayload * depay, GstEvent * event) { - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (depay); - const GstStructure *s; - gboolean might_have_been_fec; - gboolean unref_event = FALSE; - gboolean ret; - - s = gst_event_get_structure (event); - + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (depay); if (self->stop_lost_events) { - if (gst_structure_get_boolean (s, "might-have-been-fec", - &might_have_been_fec) - && might_have_been_fec) { - GST_DEBUG_OBJECT (depay, "Stopping lost event %" GST_PTR_FORMAT, event); - gst_event_replace (&self->last_lost_event, event); - return TRUE; - } - } else if (self->last_picture_id != PICTURE_ID_NONE) { - GstStructure *s; - - if (!gst_event_is_writable (event)) { - event = gst_event_copy (event); - unref_event = TRUE; - } - - s = gst_event_writable_structure (event); - - /* We are currently processing a picture, let's make sure the - * base depayloader doesn't drop this lost event */ - gst_structure_remove_field (s, "might-have-been-fec"); + GST_DEBUG_OBJECT (depay, "Stopping lost event %" GST_PTR_FORMAT, event); + if (self->last_lost_event) + gst_event_unref (self->last_lost_event); + self->last_lost_event = gst_event_ref (event); + return TRUE; } self->last_pushed_was_lost_event = TRUE; - ret = + return GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp8_depay_parent_class)->packet_lost (depay, event); - - if (unref_event) - gst_event_unref (event); - - return ret; -} +} \ No newline at end of file diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.h index 76be52adc7b..388ce88cb2b 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.h @@ -24,7 +24,6 @@ #include G_BEGIN_DECLS - #define GST_TYPE_RTP_VP8_DEPAY \ (gst_rtp_vp8_depay_get_type()) #define GST_RTP_VP8_DEPAY(obj) \ @@ -39,6 +38,7 @@ G_BEGIN_DECLS #define GST_RTP_VP8_DEPAY_GET_CLASS(obj) \ (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_RTP_VP8_DEPAY, \ GstRtpVP8DepayClass)) +#define GST_RTP_VP8_DEPAY_CAST(obj) ((GstRtpVP8Depay *)(obj)) typedef struct _GstRtpVP8Depay GstRtpVP8Depay; typedef struct _GstRtpVP8DepayClass GstRtpVP8DepayClass; @@ -55,23 +55,18 @@ struct _GstRtpVP8Depay gboolean started; gboolean caps_sent; - /* In between pictures, we might store GstRTPPacketLost events instead - * of forwarding them immediately, we check upon reception of a new - * picture id whether a gap was introduced, in which case we do forward - * the event. This is to avoid forwarding spurious lost events for FEC - * packets. - */ gboolean stop_lost_events; GstEvent *last_lost_event; gboolean waiting_for_keyframe; + gboolean last_pushed_was_lost_event; gint last_profile; gint last_width; gint last_height; guint last_picture_id; + /* properties */ gboolean wait_for_keyframe; - gboolean request_keyframe; - gboolean last_pushed_was_lost_event; + gboolean hide_picture_id_gap; }; GType gst_rtp_vp8_depay_get_type (void); From b56a04d2aaaaf4efc8d379665c830a8b94b3e36c Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 27 Aug 2021 14:59:26 +0200 Subject: [PATCH 160/339] vp9enc: disable VP9E_SET_COLOR_SPACE for now ...it crashes our libvpx... --- subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c index 3ee716ed4b1..22608a17f5a 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c @@ -468,6 +468,7 @@ gst_vp9_enc_configure_encoder (GstVPXEnc * encoder, GstVideoCodecState * state) GstVideoInfo *info = &state->info; vpx_codec_err_t status; +#if 0 status = vpx_codec_control (&encoder->encoder, VP9E_SET_COLOR_SPACE, gst_vp9_get_vpx_colorspace (encoder, &GST_VIDEO_INFO_COLORIMETRY (info), GST_VIDEO_INFO_FORMAT (info))); @@ -475,6 +476,7 @@ gst_vp9_enc_configure_encoder (GstVPXEnc * encoder, GstVideoCodecState * state) GST_WARNING_OBJECT (encoder, "Failed to set VP9E_SET_COLOR_SPACE: %s", gst_vpx_error_name (status)); } +#endif status = vpx_codec_control (&encoder->encoder, VP9E_SET_COLOR_RANGE, gst_vp9_get_vpx_color_range (&GST_VIDEO_INFO_COLORIMETRY (info))); From d3fad19cbd3f30a82e08e21f9edd862d925cb45a Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 2 Sep 2021 13:54:25 +0200 Subject: [PATCH 161/339] rtpvp9depay: revert to pexip-changes (for now) --- .../gst-plugins-good/gst/rtp/gstrtpvp9depay.c | 303 ++++++++---------- .../gst-plugins-good/gst/rtp/gstrtpvp9depay.h | 12 +- 2 files changed, 147 insertions(+), 168 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.c index 5c5f71c76c2..f1ae8461286 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.c @@ -35,10 +35,6 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_vp9_depay_debug); #define GST_CAT_DEFAULT gst_rtp_vp9_depay_debug static void gst_rtp_vp9_depay_dispose (GObject * object); -static void gst_rtp_vp9_depay_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec); -static void gst_rtp_vp9_depay_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec); static GstBuffer *gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp); static GstStateChangeReturn gst_rtp_vp9_depay_change_state (GstElement * @@ -67,30 +63,55 @@ GST_STATIC_PAD_TEMPLATE ("sink", "media = (string) \"video\"," "encoding-name = (string) { \"VP9\", \"VP9-DRAFT-IETF-01\" }")); -#define DEFAULT_WAIT_FOR_KEYFRAME FALSE -#define DEFAULT_REQUEST_KEYFRAME FALSE +#define PICTURE_ID_NONE (UINT_MAX) +#define IS_PICTURE_ID_15BITS(pid) (((guint)(pid) & 0x8000) != 0) +#define PICTURE_ID_WITHOUT_MBIT(pid) ((pid) & 0x7fff) + +#define DEFAULT_HIDE_PICTURE_ID_GAP FALSE enum { PROP_0, - PROP_WAIT_FOR_KEYFRAME, - PROP_REQUEST_KEYFRAME, + PROP_HIDE_PICTURE_ID_GAP, }; -#define PICTURE_ID_NONE (UINT_MAX) -#define IS_PICTURE_ID_15BITS(pid) (((guint)(pid) & 0x8000) != 0) +static void +gst_rtp_vp9_depay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY_CAST (object); + switch (prop_id) { + case PROP_HIDE_PICTURE_ID_GAP: + self->hide_picture_id_gap = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} static void -gst_rtp_vp9_depay_init (GstRtpVP9Depay * self) +gst_rtp_vp9_depay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) { - gst_rtp_base_depayload_set_aggregate_hdrext_enabled (GST_RTP_BASE_DEPAYLOAD - (self), TRUE); + GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY_CAST (object); + switch (prop_id) { + case PROP_HIDE_PICTURE_ID_GAP: + g_value_set_boolean (value, self->hide_picture_id_gap); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} +static void +gst_rtp_vp9_depay_init (GstRtpVP9Depay * self) +{ self->adapter = gst_adapter_new (); self->started = FALSE; self->inter_picture = FALSE; - self->wait_for_keyframe = DEFAULT_WAIT_FOR_KEYFRAME; - self->request_keyframe = DEFAULT_REQUEST_KEYFRAME; + self->last_pushed_was_lost_event = FALSE; } static void @@ -111,35 +132,9 @@ gst_rtp_vp9_depay_class_init (GstRtpVP9DepayClass * gst_rtp_vp9_depay_class) "Codec/Depayloader/Network/RTP", "Extracts VP9 video from RTP packets)", "Stian Selnes "); - object_class->dispose = gst_rtp_vp9_depay_dispose; object_class->set_property = gst_rtp_vp9_depay_set_property; object_class->get_property = gst_rtp_vp9_depay_get_property; - - /** - * GstRtpVP9Depay:wait-for-keyframe: - * - * Wait for the next keyframe after packet loss - * - * Since: 1.22 - */ - g_object_class_install_property (object_class, PROP_WAIT_FOR_KEYFRAME, - g_param_spec_boolean ("wait-for-keyframe", "Wait for Keyframe", - "Wait for the next keyframe after packet loss", - DEFAULT_WAIT_FOR_KEYFRAME, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - /** - * GstRtpVP9Depay:request-keyframe: - * - * Request new keyframe when packet loss is detected - * - * Since: 1.22 - */ - g_object_class_install_property (object_class, PROP_REQUEST_KEYFRAME, - g_param_spec_boolean ("request-keyframe", "Request Keyframe", - "Request new keyframe when packet loss is detected", - DEFAULT_REQUEST_KEYFRAME, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + object_class->dispose = gst_rtp_vp9_depay_dispose; element_class->change_state = gst_rtp_vp9_depay_change_state; @@ -147,6 +142,12 @@ gst_rtp_vp9_depay_class_init (GstRtpVP9DepayClass * gst_rtp_vp9_depay_class) depay_class->handle_event = gst_rtp_vp9_depay_handle_event; depay_class->packet_lost = gst_rtp_vp9_depay_packet_lost; + g_object_class_install_property (object_class, PROP_HIDE_PICTURE_ID_GAP, + g_param_spec_boolean ("hide-picture-id-gap", "Hide Picture ID Gap", + "Wether to trigger a key-unit request when there is a gap in " + "the picture ID", DEFAULT_HIDE_PICTURE_ID_GAP, + G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + GST_DEBUG_CATEGORY_INIT (gst_rtp_vp9_depay_debug, "rtpvp9depay", 0, "VP9 Video RTP Depayloader"); } @@ -166,45 +167,6 @@ gst_rtp_vp9_depay_dispose (GObject * object) G_OBJECT_CLASS (gst_rtp_vp9_depay_parent_class)->dispose (object); } -static void -gst_rtp_vp9_depay_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) -{ - GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY (object); - - switch (prop_id) { - case PROP_WAIT_FOR_KEYFRAME: - self->wait_for_keyframe = g_value_get_boolean (value); - break; - case PROP_REQUEST_KEYFRAME: - self->request_keyframe = g_value_get_boolean (value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static void -gst_rtp_vp9_depay_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY (object); - - switch (prop_id) { - case PROP_WAIT_FOR_KEYFRAME: - g_value_set_boolean (value, self->wait_for_keyframe); - break; - case PROP_REQUEST_KEYFRAME: - g_value_set_boolean (value, self->request_keyframe); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - - static gint picture_id_compare (guint16 id0, guint16 id1) { @@ -224,46 +186,86 @@ send_last_lost_event (GstRtpVP9Depay * self) GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp9_depay_parent_class) ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), self->last_lost_event); - gst_event_replace (&self->last_lost_event, NULL); + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + self->last_pushed_was_lost_event = TRUE; } } static void -send_last_lost_event_if_needed (GstRtpVP9Depay * self, guint new_picture_id) +send_new_lost_event (GstRtpVP9Depay * self, GstClockTime timestamp, + guint new_picture_id, const gchar * reason) { + GstEvent *event; + + if (!GST_CLOCK_TIME_IS_VALID (timestamp)) { + GST_WARNING_OBJECT (self, + "Can't create lost event with invalid timestmap"); + return; + } + + event = gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new ("GstRTPPacketLost", + "timestamp", G_TYPE_UINT64, timestamp, + "duration", G_TYPE_UINT64, 0, + "no-packet-loss", G_TYPE_BOOLEAN, self->hide_picture_id_gap, + NULL)); + + GST_DEBUG_OBJECT (self, "Pushing lost event " + "(picids 0x%x 0x%x, reason \"%s\"): %" GST_PTR_FORMAT, + self->last_picture_id, new_picture_id, reason, event); + + GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp9_depay_parent_class) + ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), event); + + gst_event_unref (event); +} + +static void +send_lost_event_if_needed (GstRtpVP9Depay * self, guint new_picture_id, + GstClockTime lost_event_timestamp) +{ + const gchar *reason = NULL; + gboolean fwd_last_lost_event = FALSE; + gboolean create_lost_event = FALSE; + if (self->last_picture_id == PICTURE_ID_NONE || self->last_picture_id == new_picture_id) return; + if (new_picture_id == PICTURE_ID_NONE) { + reason = "picture id does not exist"; + fwd_last_lost_event = TRUE; + } else if (IS_PICTURE_ID_15BITS (self->last_picture_id) && + !IS_PICTURE_ID_15BITS (new_picture_id)) { + reason = "picture id has less bits than before"; + fwd_last_lost_event = TRUE; + } else if (picture_id_compare (self->last_picture_id, new_picture_id) != 1) { + reason = "picture id gap"; + fwd_last_lost_event = TRUE; + /* Only create a new one if we just didn't push a lost event */ + create_lost_event = self->last_pushed_was_lost_event == FALSE; + } + if (self->last_lost_event) { - gboolean send_lost_event = FALSE; - if (new_picture_id == PICTURE_ID_NONE) { - GST_DEBUG_OBJECT (self, "Dropping the last stopped lost event " - "(picture id does not exist): %" GST_PTR_FORMAT, - self->last_lost_event); - } else if (IS_PICTURE_ID_15BITS (self->last_picture_id) && - !IS_PICTURE_ID_15BITS (new_picture_id)) { - GST_DEBUG_OBJECT (self, "Dropping the last stopped lost event " - "(picture id has less bits than before): %" GST_PTR_FORMAT, - self->last_lost_event); - } else if (picture_id_compare (self->last_picture_id, new_picture_id) != 1) { - GstStructure *s = gst_event_writable_structure (self->last_lost_event); - - GST_DEBUG_OBJECT (self, "Sending the last stopped lost event " - "(gap in picture id %u %u): %" GST_PTR_FORMAT, - self->last_picture_id, new_picture_id, self->last_lost_event); - send_lost_event = TRUE; - /* Prevent rtpbasedepayload from dropping the event now - * that we have made sure the lost packet was not FEC */ - gst_structure_remove_field (s, "might-have-been-fec"); - } - if (send_lost_event) + if (fwd_last_lost_event) { + GST_DEBUG_OBJECT (self, "Forwarding lost event " + "(picids 0x%x 0x%x, reason \"%s\"): %" GST_PTR_FORMAT, + self->last_picture_id, new_picture_id, reason, self->last_lost_event); GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp9_depay_parent_class) ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), self->last_lost_event); - gst_event_replace (&self->last_lost_event, NULL); + // If we forward last received lost event, there is no need + // to create another one + create_lost_event = FALSE; + } + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; } + + if (create_lost_event) + send_new_lost_event (self, lost_event_timestamp, new_picture_id, reason); } static GstBuffer * @@ -278,13 +280,12 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) guint picture_id = PICTURE_ID_NONE; gboolean i_bit, p_bit, l_bit, f_bit, b_bit, e_bit, v_bit, d_bit = 0; gboolean is_start_of_picture; - gboolean flushed_adapter = FALSE; + gboolean sent_lost_event = FALSE; if (G_UNLIKELY (GST_BUFFER_IS_DISCONT (rtp->buffer))) { GST_LOG_OBJECT (self, "Discontinuity, flushing adapter"); gst_adapter_clear (self->adapter); self->started = FALSE; - flushed_adapter = TRUE; } size = gst_rtp_buffer_get_payload_len (rtp); @@ -424,38 +425,22 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) if (is_start_of_picture) { if (G_UNLIKELY (self->started)) { GST_DEBUG_OBJECT (depay, "Incomplete frame, flushing adapter"); - /* keep the current buffer because it may still be used later */ - gst_rtp_base_depayload_flush (depay, TRUE); gst_adapter_clear (self->adapter); self->started = FALSE; - flushed_adapter = TRUE; + + /* FIXME: Add property to control whether to send GAP events */ + send_new_lost_event (self, GST_BUFFER_PTS (rtp->buffer), picture_id, + "Incomplete frame detected"); + sent_lost_event = TRUE; } } if (G_UNLIKELY (!self->started)) { - self->inter_picture = FALSE; - - /* We have flushed the adapter and this packet does not - * start a keyframe, request one if needed */ - if (flushed_adapter && (!b_bit || p_bit)) { - if (self->wait_for_keyframe) { - GST_DEBUG_OBJECT (self, "Waiting for keyframe after flushing adapter"); - self->waiting_for_keyframe = TRUE; - } - - if (self->request_keyframe) { - GST_DEBUG_OBJECT (self, "Requesting keyframe after flushing adapter"); - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); - } - } - /* Check if this is the start of a VP9 layer frame, otherwise bail */ if (!b_bit) { GST_DEBUG_OBJECT (depay, "The layer is missing the first packets, ignoring the packet"); - if (self->stop_lost_events) { + if (self->stop_lost_events && !sent_lost_event) { send_last_lost_event (self); self->stop_lost_events = FALSE; } @@ -463,11 +448,11 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) } GST_DEBUG_OBJECT (depay, "Found the start of the layer"); - if (self->stop_lost_events) { - send_last_lost_event_if_needed (self, picture_id); - self->stop_lost_events = FALSE; - } + if (!sent_lost_event) + send_lost_event_if_needed (self, picture_id, GST_BUFFER_PTS (rtp->buffer)); self->started = TRUE; + self->stop_lost_events = FALSE; + self->inter_picture = FALSE; } payload = gst_rtp_buffer_get_payload_subbuffer (rtp, hdrsize, -1); @@ -507,8 +492,7 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) if (self->inter_picture) { GST_BUFFER_FLAG_SET (out, GST_BUFFER_FLAG_DELTA_UNIT); - if (self->waiting_for_keyframe) { - gst_rtp_base_depayload_flush (depay, FALSE); + if (!self->caps_sent) { gst_buffer_unref (out); out = NULL; GST_INFO_OBJECT (self, "Dropping inter-frame before intra-frame"); @@ -538,27 +522,27 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (depay), srccaps); gst_caps_unref (srccaps); + self->caps_sent = TRUE; self->last_width = self->ss_width; self->last_height = self->ss_height; self->ss_width = 0; self->ss_height = 0; } - - self->waiting_for_keyframe = FALSE; } if (picture_id != PICTURE_ID_NONE) self->stop_lost_events = TRUE; + + self->last_pushed_was_lost_event = FALSE; + return out; } done: - gst_rtp_base_depayload_dropped (depay); return NULL; too_small: GST_LOG_OBJECT (self, "Invalid rtp packet (too small), ignoring"); - gst_rtp_base_depayload_flush (depay, FALSE); gst_adapter_clear (self->adapter); self->started = FALSE; goto done; @@ -573,10 +557,13 @@ gst_rtp_vp9_depay_change_state (GstElement * element, GstStateChange transition) case GST_STATE_CHANGE_READY_TO_PAUSED: self->last_width = -1; self->last_height = -1; + self->caps_sent = FALSE; self->last_picture_id = PICTURE_ID_NONE; - gst_event_replace (&self->last_lost_event, NULL); + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } self->stop_lost_events = FALSE; - self->waiting_for_keyframe = TRUE; break; default: break; @@ -597,7 +584,10 @@ gst_rtp_vp9_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) self->last_width = -1; self->last_height = -1; self->last_picture_id = PICTURE_ID_NONE; - gst_event_replace (&self->last_lost_event, NULL); + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } self->stop_lost_events = FALSE; break; default: @@ -613,28 +603,17 @@ static gboolean gst_rtp_vp9_depay_packet_lost (GstRTPBaseDepayload * depay, GstEvent * event) { GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY (depay); - const GstStructure *s; - gboolean might_have_been_fec; - - s = gst_event_get_structure (event); - if (self->stop_lost_events) { - if (gst_structure_get_boolean (s, "might-have-been-fec", - &might_have_been_fec) - && might_have_been_fec) { - GST_DEBUG_OBJECT (depay, "Stopping lost event %" GST_PTR_FORMAT, event); - gst_event_replace (&self->last_lost_event, event); - return TRUE; - } - } else if (self->last_picture_id != PICTURE_ID_NONE) { - GstStructure *s = gst_event_writable_structure (self->last_lost_event); - - /* We are currently processing a picture, let's make sure the - * base depayloader doesn't drop this lost event */ - gst_structure_remove_field (s, "might-have-been-fec"); + GST_DEBUG_OBJECT (depay, "Stopping lost event %" GST_PTR_FORMAT, event); + if (self->last_lost_event) + gst_event_unref (self->last_lost_event); + self->last_lost_event = gst_event_ref (event); + return TRUE; } + self->last_pushed_was_lost_event = TRUE; + return GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp9_depay_parent_class)->packet_lost (depay, event); -} +} \ No newline at end of file diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.h index 8c98a5bf186..fd5bbceb3f3 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.h @@ -40,6 +40,7 @@ G_BEGIN_DECLS #define GST_RTP_VP9_DEPAY_GET_CLASS(obj) \ (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_RTP_VP9_DEPAY, \ GstRtpVP9DepayClass)) +#define GST_RTP_VP9_DEPAY_CAST(obj) ((GstRtpVP9Depay *)(obj)) typedef struct _GstRtpVP9Depay GstRtpVP9Depay; typedef struct _GstRtpVP9DepayClass GstRtpVP9DepayClass; @@ -61,6 +62,8 @@ struct _GstRtpVP9Depay gint last_height; guint last_picture_id; GstEvent *last_lost_event; + gboolean last_pushed_was_lost_event; + gboolean caps_sent; /* In between pictures, we might store GstRTPPacketLost events instead * of forwarding them immediately, we check upon reception of a new * picture id whether a gap was introduced, in which case we do forward @@ -70,15 +73,12 @@ struct _GstRtpVP9Depay gboolean stop_lost_events; gboolean inter_picture; - gboolean waiting_for_keyframe; - - /* Properties */ - gboolean wait_for_keyframe; - gboolean request_keyframe; + /* properties */ + gboolean hide_picture_id_gap; }; GType gst_rtp_vp9_depay_get_type (void); G_END_DECLS -#endif /* #ifndef __GST_RTP_VP9_DEPAY_H__ */ +#endif /* #ifndef __GST_RTP_VP9_DEPAY_H__ */ \ No newline at end of file From 27d8bed0845daf0b941abaf974726ce90ad2e371 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 20 Sep 2021 14:28:41 +0200 Subject: [PATCH 162/339] rtph261depay: don't set the DELTA_UNIT flag for H.261 Basically, all frames are "i-frames" here. --- subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c index 23a888a4eac..3d0af365b94 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c @@ -178,8 +178,8 @@ gst_rtp_h261_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) * stream is intra coded. */ if (header->i) GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); - else - GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + //else + // GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); GST_DEBUG_OBJECT (depay, "Pushing out a buffer of %u bytes", avail); depay->start = FALSE; From 438f1d38d57889871cd844e06d86ce792cebc5f5 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 20 Sep 2021 18:35:50 +0200 Subject: [PATCH 163/339] vpxenc: move temporal-scalability-layer to GValueArray Can't have a mix of GstValueArray and GValueArray here. --- .../gst-plugins-good/ext/vpx/gstvpxenc.c | 83 +++++++++++-------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c index d17219dc666..0777aded457 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c @@ -602,13 +602,13 @@ gst_vpx_enc_class_init (GstVPXEncClass * klass) * Since: 1.20 */ g_object_class_install_property (gobject_class, PROP_TS_LAYER_FLAGS, - gst_param_spec_array ("temporal-scalability-layer-flags", + g_param_spec_value_array ("temporal-scalability-layer-flags", "Coding layer flags", "Sequence defining coding layer flags", - g_param_spec_flags ("flags", "Flags", "Flags", - GST_VPX_ENC_TS_LAYER_FLAGS_TYPE, 0, + g_param_spec_int ("flags", "Flags", "Flags", 0, G_MAXINT, 0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS), G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** * GstVPXEnc:temporal-scalability-layer-sync-flags: * @@ -617,7 +617,7 @@ gst_vpx_enc_class_init (GstVPXEncClass * klass) * Since: 1.20 */ g_object_class_install_property (gobject_class, PROP_TS_LAYER_SYNC_FLAGS, - gst_param_spec_array ("temporal-scalability-layer-sync-flags", + g_param_spec_value_array ("temporal-scalability-layer-sync-flags", "Coding layer sync flags", "Sequence defining coding layer sync flags", g_param_spec_boolean ("flags", "Flags", "Flags", FALSE, @@ -1105,41 +1105,36 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, break; } case PROP_TS_LAYER_FLAGS:{ - gint l = gst_value_array_get_size (value); + GValueArray *va = g_value_get_boxed (value); g_free (gst_vpx_enc->ts_layer_flags); gst_vpx_enc->n_ts_layer_flags = 0; - if (l > 0) { + if (va && va->n_values > 0) { gint i; - gst_vpx_enc->ts_layer_flags = g_new (gint, l); - - for (i = 0; i < l; i++) + gst_vpx_enc->ts_layer_flags = g_new (gint, va->n_values); + for (i = 0; i < va->n_values; i++) gst_vpx_enc->ts_layer_flags[i] = - g_value_get_flags (gst_value_array_get_value (value, i)); - gst_vpx_enc->n_ts_layer_flags = l; - } else { - gst_vpx_enc->ts_layer_flags = NULL; + g_value_get_int (g_value_array_get_nth (va, i)); + gst_vpx_enc->n_ts_layer_flags = va->n_values; } break; } case PROP_TS_LAYER_SYNC_FLAGS:{ - gint l = gst_value_array_get_size (value); + GValueArray *va = g_value_get_boxed (value); g_free (gst_vpx_enc->ts_layer_sync_flags); gst_vpx_enc->n_ts_layer_sync_flags = 0; - if (l > 0) { + if (va && va->n_values > 0) { gint i; - gst_vpx_enc->ts_layer_sync_flags = g_new (gboolean, l); - for (i = 0; i < l; i++) + gst_vpx_enc->ts_layer_sync_flags = g_new (gboolean, va->n_values); + for (i = 0; i < va->n_values; i++) gst_vpx_enc->ts_layer_sync_flags[i] = - g_value_get_boolean (gst_value_array_get_value (value, i)); - gst_vpx_enc->n_ts_layer_sync_flags = l; - } else { - gst_vpx_enc->ts_layer_sync_flags = NULL; + g_value_get_boolean (g_value_array_get_nth (va, i)); + gst_vpx_enc->n_ts_layer_sync_flags = va->n_values; } break; } @@ -1538,28 +1533,46 @@ gst_vpx_enc_get_property (GObject * object, guint prop_id, GValue * value, break; } case PROP_TS_LAYER_FLAGS:{ - gint i; + GValueArray *va; + + if (gst_vpx_enc->n_ts_layer_flags == 0) { + g_value_set_boxed (value, NULL); + } else { + gint i; - for (i = 0; i < gst_vpx_enc->n_ts_layer_flags; i++) { - GValue v = { 0, }; + va = g_value_array_new (gst_vpx_enc->n_ts_layer_flags); + for (i = 0; i < gst_vpx_enc->n_ts_layer_flags; i++) { + GValue v = { 0, }; - g_value_init (&v, GST_VPX_ENC_TS_LAYER_FLAGS_TYPE); - g_value_set_flags (&v, gst_vpx_enc->ts_layer_flags[i]); - gst_value_array_append_value (value, &v); - g_value_unset (&v); + g_value_init (&v, G_TYPE_INT); + g_value_set_int (&v, gst_vpx_enc->ts_layer_flags[i]); + g_value_array_append (va, &v); + g_value_unset (&v); + } + g_value_set_boxed (value, va); + g_value_array_free (va); } break; } case PROP_TS_LAYER_SYNC_FLAGS:{ - gint i; + GValueArray *va; + + if (gst_vpx_enc->n_ts_layer_sync_flags == 0) { + g_value_set_boxed (value, NULL); + } else { + gint i; - for (i = 0; i < gst_vpx_enc->n_ts_layer_sync_flags; i++) { - GValue v = { 0, }; + va = g_value_array_new (gst_vpx_enc->n_ts_layer_sync_flags); + for (i = 0; i < gst_vpx_enc->n_ts_layer_sync_flags; i++) { + GValue v = { 0, }; - g_value_init (&v, G_TYPE_BOOLEAN); - g_value_set_boolean (&v, gst_vpx_enc->ts_layer_sync_flags[i]); - gst_value_array_append_value (value, &v); - g_value_unset (&v); + g_value_init (&v, G_TYPE_BOOLEAN); + g_value_set_boolean (&v, gst_vpx_enc->ts_layer_sync_flags[i]); + g_value_array_append (va, &v); + g_value_unset (&v); + } + g_value_set_boxed (value, va); + g_value_array_free (va); } break; } From 44ad64cea65d88b539630573cea0645f09662b2c Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 24 Oct 2019 22:32:35 +0200 Subject: [PATCH 164/339] [pexhack] find out about -objc-arc stuff --- subprojects/gst-plugins-bad/sys/applemedia/meson.build | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/meson.build b/subprojects/gst-plugins-bad/sys/applemedia/meson.build index ebe839fb8ad..5c285cec5ab 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/meson.build +++ b/subprojects/gst-plugins-bad/sys/applemedia/meson.build @@ -29,9 +29,9 @@ endif if ['darwin', 'ios'].contains(host_system) objc = meson.get_compiler('objc') - if not objc.has_argument('-fobjc-arc') - error('ARC is required for building') - endif +# if not objc.has_argument('-fobjc-arc') +# error('ARC is required for building') +# endif applemedia_objc_args += ['-fobjc-arc'] From 6a85b2425b18229b3913998a3c455b32d79cc663 Mon Sep 17 00:00:00 2001 From: Tulio Date: Fri, 26 Jun 2020 08:05:41 -0300 Subject: [PATCH 165/339] gstgdiscreencapsrc: added window property --- .../sys/winscreencap/gstgdiscreencapsrc.c | 33 +++++++++++++++++-- .../sys/winscreencap/gstgdiscreencapsrc.h | 1 + 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c index a398bd643db..e1771eaead6 100644 --- a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c +++ b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c @@ -63,6 +63,7 @@ enum { PROP_0, PROP_MONITOR, + PROP_WINDOW, PROP_SHOW_CURSOR, PROP_X_POS, PROP_Y_POS, @@ -122,6 +123,11 @@ gst_gdiscreencapsrc_class_init (GstGDIScreenCapSrcClass * klass) "Monitor", "Which monitor to use (0 = 1st monitor and default)", 0, G_MAXINT, 0, G_PARAM_READWRITE)); + g_object_class_install_property (go_class, PROP_WINDOW, + g_param_spec_int ("window", "Window", + "The window handle (HWND) to capture.", 0, G_MAXINT, 0, + G_PARAM_READWRITE)); + g_object_class_install_property (go_class, PROP_SHOW_CURSOR, g_param_spec_boolean ("cursor", "Show mouse cursor", "Whether to show mouse cursor (default off)", @@ -169,6 +175,7 @@ gst_gdiscreencapsrc_init (GstGDIScreenCapSrc * src) src->capture_h = 0; src->monitor = 0; + src->window = 0; src->show_cursor = FALSE; gst_base_src_set_format (GST_BASE_SRC (src), GST_FORMAT_TIME); @@ -215,6 +222,9 @@ gst_gdiscreencapsrc_set_property (GObject * object, guint prop_id, } src->monitor = g_value_get_int (value); break; + case PROP_WINDOW: + src->window = g_value_get_int (value); + break; case PROP_SHOW_CURSOR: src->show_cursor = g_value_get_boolean (value); break; @@ -246,6 +256,9 @@ gst_gdiscreencapsrc_get_property (GObject * object, guint prop_id, case PROP_MONITOR: g_value_set_int (value, src->monitor); break; + case PROP_WINDOW: + g_value_set_int (value, src->window); + break; case PROP_SHOW_CURSOR: g_value_set_boolean (value, src->show_cursor); break; @@ -330,7 +343,7 @@ gst_gdiscreencapsrc_set_caps (GstBaseSrc * bsrc, GstCaps * caps) DeleteDC (src->memDC); /* Allocate */ - capture = GetDesktopWindow (); + capture = src->window > 0 ? (HWND) src->window : GetDesktopWindow (); device = GetDC (capture); src->hBitmap = CreateDIBSection (device, &(src->info), DIB_RGB_COLORS, (void **) &(src->dibMem), 0, 0); @@ -346,6 +359,17 @@ gst_gdiscreencapsrc_set_caps (GstBaseSrc * bsrc, GstCaps * caps) return TRUE; } +static RECT +gst_win32_get_window_rect (HWND wnd) +{ + RECT rect; + + if (!GetClientRect (wnd, &rect)) + ZeroMemory (&rect, sizeof (RECT)); + + return rect; +} + static GstCaps * gst_gdiscreencapsrc_get_caps (GstBaseSrc * bsrc, GstCaps * filter) { @@ -353,7 +377,10 @@ gst_gdiscreencapsrc_get_caps (GstBaseSrc * bsrc, GstCaps * filter) RECT rect_dst; GstCaps *caps; - src->screen_rect = rect_dst = gst_win32_get_monitor_rect (src->monitor); + if (src->window > 0) + src->screen_rect = rect_dst = gst_win32_get_window_rect (src->window); + else + src->screen_rect = rect_dst = gst_win32_get_monitor_rect (src->monitor); if (src->capture_w && src->capture_h && src->capture_x + src->capture_w < rect_dst.right - rect_dst.left && @@ -539,7 +566,7 @@ gst_gdiscreencapsrc_screen_capture (GstGDIScreenCapSrc * src, GstBuffer * buf) height = -src->info.bmiHeader.biHeight; /* Capture screen */ - capture = GetDesktopWindow (); + capture = src->window > 0 ? (HWND) src->window : GetDesktopWindow (); winDC = GetWindowDC (capture); BitBlt (src->memDC, 0, 0, width, height, diff --git a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.h b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.h index 085e5328866..6b6516430fd 100644 --- a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.h +++ b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.h @@ -53,6 +53,7 @@ struct _GstGDIScreenCapSrc gint capture_w; gint capture_h; gint monitor; + gint window; gboolean show_cursor; /* Source pad frame rate */ From 970f5729df68ebc0918ea6208f687fd381ea7411 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Fri, 15 Mar 2019 21:31:23 +0000 Subject: [PATCH 166/339] SCTP: expose gi bindings --- .../gst-libs/gst/sctp/meson.build | 18 +++++++++++++ .../gst-libs/gst/sctp/sctpreceivemeta.c | 25 +++++++++++++++++++ .../gst-libs/gst/sctp/sctpreceivemeta.h | 4 +-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/meson.build index 2c0a48cf180..6ffa3942560 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/meson.build +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/meson.build @@ -31,8 +31,26 @@ pkgconfig.generate(libgstsctp, description : 'SCTP helper functions', ) +sctp_gen_sources = [] +if build_gir + sctp_gir = gnome.generate_gir(libgstsctp, + sources : sctp_sources + sctp_headers, + namespace : 'GstSctp', + nsversion : api_version, + identifier_prefix : 'Gst', + symbol_prefix : 'gst', + export_packages : 'gstreamer-sctp-1.0', + includes : ['Gst-1.0'], + install : true, + extra_args : gir_init_section + ['-DGST_USE_UNSTABLE_API'], + dependencies : [gstbase_dep] + ) + sctp_gen_sources += sctp_gir +endif + gstsctp_dep = declare_dependency(link_with : libgstsctp, include_directories : [libsinc], + sources : sctp_gen_sources, dependencies : [gstbase_dep]) meson.override_dependency(pkg_name, gstsctp_dep) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.c b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.c index 11def65238d..4a21b74ca01 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.c @@ -79,6 +79,15 @@ gst_sctp_receive_meta_transform (GstBuffer * transbuf, GstMeta * meta, return TRUE; } +/** + * gst_sctp_buffer_add_receive_meta: + * @buffer: a #GstBuffer + * @ppid: sctp ppid + * + * Attaches @ppid as metadata in a #GstSctpReceiveMeta to @buffer. + * + * Returns: (transfer none): a #GstSctpReceiveMeta connected to @buffer + */ GstSctpReceiveMeta * gst_sctp_buffer_add_receive_meta (GstBuffer * buffer, guint32 ppid) { @@ -91,3 +100,19 @@ gst_sctp_buffer_add_receive_meta (GstBuffer * buffer, guint32 ppid) gst_sctp_receive_meta->ppid = ppid; return gst_sctp_receive_meta; } + +/** + * gst_sctp_buffer_get_receive_meta: + * @buffer: a #GstBuffer + * + * Find the #GstSctpReceiveMeta on @buffer. + * + * Returns: (transfer none): the #GstSctpReceiveMeta or %NULL when there + * is no such metadata on @buffer. + */ +GstSctpReceiveMeta * +gst_sctp_buffer_get_receive_meta (GstBuffer * buffer) +{ + return (GstSctpReceiveMeta *) + gst_buffer_get_meta(buffer, GST_SCTP_RECEIVE_META_API_TYPE); +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.h b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.h index 5a508370c2f..e8f2f488319 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.h +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.h @@ -49,8 +49,8 @@ const GstMetaInfo *gst_sctp_receive_meta_get_info (void); GST_SCTP_API GstSctpReceiveMeta *gst_sctp_buffer_add_receive_meta (GstBuffer * buffer, guint32 ppid); - -#define gst_sctp_buffer_get_receive_meta(b) ((GstSctpReceiveMeta *)gst_buffer_get_meta((b), GST_SCTP_RECEIVE_META_API_TYPE)) +GST_SCTP_API +GstSctpReceiveMeta *gst_sctp_buffer_get_receive_meta (GstBuffer * buffer); G_END_DECLS From 042bde313521dd8bb3fb0ec40f27881c7457da30 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Tue, 26 Mar 2019 20:34:09 +0000 Subject: [PATCH 167/339] SCTP: expose gi bindings for send meta --- .../gst-libs/gst/sctp/sctpsendmeta.c | 28 +++++++++++++++++++ .../gst-libs/gst/sctp/sctpsendmeta.h | 4 +-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.c b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.c index e29cd7d9273..9b18c9c3ef0 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.c @@ -84,6 +84,18 @@ gst_sctp_send_meta_transform (GstBuffer * transbuf, GstMeta * meta, return TRUE; } +/** + * gst_sctp_buffer_add_send_meta: + * @buffer: a #GstBuffer + * @ppid: sctp ppid + * @ordered: whether buffer should be sent ordered + * @pr: partial reliability mode + * @pr_param: partial reliability parameter + * + * Attaches metadata in a #GstSctpSendMeta to @buffer. + * + * Returns: (transfer none): a #GstSctpSendMeta connected to @buffer + */ GstSctpSendMeta * gst_sctp_buffer_add_send_meta (GstBuffer * buffer, guint32 ppid, gboolean ordered, GstSctpSendMetaPartiallyReliability pr, guint32 pr_param) @@ -100,3 +112,19 @@ gst_sctp_buffer_add_send_meta (GstBuffer * buffer, guint32 ppid, gst_sctp_send_meta->pr_param = pr_param; return gst_sctp_send_meta; } + +/** + * gst_sctp_buffer_get_send_meta: + * @buffer: a #GstBuffer + * + * Find the #GstSctpSendMeta on @buffer. + * + * Returns: (transfer none): the #GstSctpSendMeta or %NULL when there + * is no such metadata on @buffer. + */ +GstSctpSendMeta * +gst_sctp_buffer_get_send_meta (GstBuffer * buffer) +{ + return (GstSctpSendMeta *) + gst_buffer_get_meta(buffer, GST_SCTP_SEND_META_API_TYPE); +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.h b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.h index b1860e52f22..3b117a47d9e 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.h +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.h @@ -61,8 +61,8 @@ GST_SCTP_API GstSctpSendMeta *gst_sctp_buffer_add_send_meta (GstBuffer * buffer, guint32 ppid, gboolean ordered, GstSctpSendMetaPartiallyReliability pr, guint32 pr_param); - -#define gst_sctp_buffer_get_send_meta(b) ((GstSctpSendMeta *)gst_buffer_get_meta((b), GST_SCTP_SEND_META_API_TYPE)) +GST_SCTP_API +GstSctpSendMeta *gst_sctp_buffer_get_send_meta (GstBuffer * buffer); G_END_DECLS From 0d40e04427dbafa1653e07b15defa27064d11ba6 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Tue, 26 Mar 2019 22:42:54 +0000 Subject: [PATCH 168/339] SCTP: increase default streams to 65535 As per https://tools.ietf.org/html/draft-ietf-rtcweb-data-channel-13#section-6.2 --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 68c05e62f31..f86fedde22e 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -93,7 +93,7 @@ static guint signals[LAST_SIGNAL] = { 0 }; static GParamSpec *properties[NUM_PROPERTIES]; -#define DEFAULT_NUMBER_OF_SCTP_STREAMS 1024 +#define DEFAULT_NUMBER_OF_SCTP_STREAMS 65535 #define DEFAULT_LOCAL_SCTP_PORT 0 #define DEFAULT_REMOTE_SCTP_PORT 0 From f65a77ca09e27adc1c368d912edf955cef318155 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Wed, 14 Aug 2019 19:21:15 +0100 Subject: [PATCH 169/339] SCTP: remove unnecessary preprocessor definitions --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 81332904408..3b33c9dd8e4 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -28,8 +28,6 @@ #include #include -#define INET -#define INET6 #include G_BEGIN_DECLS From 80f265ff1b54e8fa28d13f12b7525f9d6a10f7e8 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Wed, 14 Aug 2019 19:24:12 +0100 Subject: [PATCH 170/339] SCTP: add property to tweak heartbeat configuration --- .../gst-plugins-bad/ext/sctp/gstsctpenc.c | 16 +++++++++++++++ .../gst-plugins-bad/ext/sctp/gstsctpenc.h | 1 + .../ext/sctp/sctpassociation.c | 20 +++++++++++++++++++ .../ext/sctp/sctpassociation.h | 1 + 4 files changed, 38 insertions(+) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 2fcbecebb9f..fbcf37450da 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -63,6 +63,7 @@ enum PROP_GST_SCTP_ASSOCIATION_ID, PROP_REMOTE_SCTP_PORT, PROP_USE_SOCK_STREAM, + PROP_AGGRESSIVE_HEARTBEAT, NUM_PROPERTIES }; @@ -215,6 +216,12 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) "When TRUE the partial reliability parameters of the channel are ignored.", DEFAULT_USE_SOCK_STREAM, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + properties[PROP_AGGRESSIVE_HEARTBEAT] = + g_param_spec_boolean ("aggressive-heartbeat", "Aggressive heartbeat", + "When set to TRUE, set the heartbeat interval to 1000ms and the assoc " + "rtx max to 2.", + FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, NUM_PROPERTIES, properties); signals[SIGNAL_SCTP_ASSOCIATION_ESTABLISHED] = @@ -303,6 +310,9 @@ gst_sctp_enc_set_property (GObject * object, guint prop_id, case PROP_USE_SOCK_STREAM: self->use_sock_stream = g_value_get_boolean (value); break; + case PROP_AGGRESSIVE_HEARTBEAT: + self->aggressive_heartbeat = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); break; @@ -325,6 +335,9 @@ gst_sctp_enc_get_property (GObject * object, guint prop_id, GValue * value, case PROP_USE_SOCK_STREAM: g_value_set_boolean (value, self->use_sock_stream); break; + case PROP_AGGRESSIVE_HEARTBEAT: + g_value_set_boolean (value, self->aggressive_heartbeat); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); break; @@ -858,6 +871,9 @@ configure_association (GstSctpEnc * self) g_object_bind_property (self, "use-sock-stream", self->sctp_association, "use-sock-stream", G_BINDING_SYNC_CREATE); + g_object_bind_property (self, "aggressive-heartbeat", self->sctp_association, + "aggressive-heartbeat", G_BINDING_SYNC_CREATE); + gst_sctp_association_set_on_packet_out (self->sctp_association, on_sctp_packet_out, gst_object_ref (self), gst_object_unref); diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index 482473d745e..71a79f9845e 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -51,6 +51,7 @@ struct _GstSctpEnc guint32 sctp_association_id; guint16 remote_sctp_port; gboolean use_sock_stream; + gboolean aggressive_heartbeat; GstSctpAssociation *sctp_association; GstDataQueue *outbound_sctp_packet_queue; diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index f86fedde22e..13b6f401cbc 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -85,6 +85,7 @@ enum PROP_REMOTE_PORT, PROP_STATE, PROP_USE_SOCK_STREAM, + PROP_AGGRESSIVE_HEARTBEAT, NUM_PROPERTIES }; @@ -171,6 +172,12 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) "When TRUE the partial reliability parameters of the channel is ignored.", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + properties[PROP_AGGRESSIVE_HEARTBEAT] = + g_param_spec_boolean ("aggressive-heartbeat", "Aggressive heartbeat", + "When set to TRUE, set the heartbeat interval to 1000ms and the assoc " + "rtx max to 2.", + FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, NUM_PROPERTIES, properties); } @@ -290,6 +297,16 @@ gst_sctp_association_set_property (GObject * object, guint prop_id, case PROP_USE_SOCK_STREAM: self->use_sock_stream = g_value_get_boolean (value); break; + case PROP_AGGRESSIVE_HEARTBEAT: + self->aggressive_heartbeat = g_value_get_boolean (value); + if (self->aggressive_heartbeat) { + usrsctp_sysctl_set_sctp_heartbeat_interval_default (1000); + usrsctp_sysctl_set_sctp_assoc_rtx_max_default (2); + } else { + usrsctp_sysctl_set_sctp_heartbeat_interval_default (30000); + usrsctp_sysctl_set_sctp_assoc_rtx_max_default (10); + } + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); break; @@ -347,6 +364,9 @@ gst_sctp_association_get_property (GObject * object, guint prop_id, case PROP_USE_SOCK_STREAM: g_value_set_boolean (value, self->use_sock_stream); break; + case PROP_AGGRESSIVE_HEARTBEAT: + g_value_set_boolean (value, self->aggressive_heartbeat); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); break; diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 3b33c9dd8e4..98ca4402e32 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -75,6 +75,7 @@ struct _GstSctpAssociation guint16 local_port; guint16 remote_port; gboolean use_sock_stream; + gboolean aggressive_heartbeat; struct socket *sctp_ass_sock; GMutex association_mutex; From 267f98caa21e5c90f83a8acddcaf02c0c4847058 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Wed, 14 Aug 2019 19:56:25 +0100 Subject: [PATCH 171/339] SCTP: change state on connection lost. --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 13b6f401cbc..6e1d8e0cd3f 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -917,8 +917,20 @@ handle_association_changed (GstSctpAssociation * self, break; case SCTP_COMM_LOST: GST_WARNING_OBJECT (self, "SCTP event SCTP_COMM_LOST received"); - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_ERROR; + /* TODO: Tear down association */ + g_mutex_lock (&self->association_mutex); + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + g_mutex_unlock (&self->association_mutex); + gst_sctp_association_change_state (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTING, TRUE); + g_mutex_lock (&self->association_mutex); + } + if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { + change_state = TRUE; + new_state = GST_SCTP_ASSOCIATION_STATE_DISCONNECTED; + GST_INFO_OBJECT (self, "SCTP association disconnected!"); + } + g_mutex_unlock (&self->association_mutex); break; case SCTP_RESTART: GST_DEBUG_OBJECT (self, "SCTP event SCTP_RESTART received"); From e7f0760c2a42a5955c137720f91684cf6df39781 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Thu, 15 Aug 2019 12:19:11 +0100 Subject: [PATCH 172/339] SCTP: change state on shutdown --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 6e1d8e0cd3f..252aa94c3cf 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -936,9 +936,21 @@ handle_association_changed (GstSctpAssociation * self, GST_DEBUG_OBJECT (self, "SCTP event SCTP_RESTART received"); break; case SCTP_SHUTDOWN_COMP: - GST_DEBUG_OBJECT (self, "SCTP event SCTP_SHUTDOWN_COMP received"); - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_DISCONNECTED; + /* Occurs if in TCP mode when the far end sends SHUTDOWN */ + GST_INFO_OBJECT (self, "SCTP event SCTP_SHUTDOWN_COMP received"); + g_mutex_lock (&self->association_mutex); + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + g_mutex_unlock (&self->association_mutex); + gst_sctp_association_change_state (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTING, TRUE); + g_mutex_lock (&self->association_mutex); + } + if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { + change_state = TRUE; + new_state = GST_SCTP_ASSOCIATION_STATE_DISCONNECTED; + GST_INFO_OBJECT (self, "SCTP association disconnected!"); + } + g_mutex_unlock (&self->association_mutex); break; case SCTP_CANT_STR_ASSOC: GST_WARNING_OBJECT (self, "SCTP event SCTP_CANT_STR_ASSOC received"); From 10e821a4eca100eb3677302c5d1797af47a520a5 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Thu, 15 Aug 2019 18:42:19 +0100 Subject: [PATCH 173/339] SCTP: record association id on connection. --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 3 +++ subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h | 2 ++ 2 files changed, 5 insertions(+) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 252aa94c3cf..9f69f189f57 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -586,6 +586,8 @@ gst_sctp_association_force_close (GstSctpAssociation * self) usrsctp_close (s); } + self->sctp_assoc_id = 0; + gst_sctp_association_change_state (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTED, TRUE); } @@ -905,6 +907,7 @@ handle_association_changed (GstSctpAssociation * self, GST_DEBUG_OBJECT (self, "SCTP_COMM_UP"); g_mutex_lock (&self->association_mutex); if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { + self->sctp_assoc_id = sac->sac_assoc_id; change_state = TRUE; new_state = GST_SCTP_ASSOCIATION_STATE_CONNECTED; GST_DEBUG_OBJECT (self, "SCTP association connected!"); diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 98ca4402e32..ddc6073fb27 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -82,6 +82,8 @@ struct _GstSctpAssociation GstSctpAssociationState state; + guint32 sctp_assoc_id; + GstSctpAssociationPacketReceivedCb packet_received_cb; gpointer packet_received_user_data; GDestroyNotify packet_received_destroy_notify; From eaecb6b87c6bc69a8b67a0c102be3887901ca023 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Thu, 15 Aug 2019 18:43:02 +0100 Subject: [PATCH 174/339] SCTP: set association id when resetting streams --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 9f69f189f57..6154fc73ca2 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -566,13 +566,15 @@ gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id) length = (socklen_t) (sizeof (struct sctp_reset_streams) + sizeof (guint16)); srs = (struct sctp_reset_streams *) g_malloc0 (length); - srs->srs_assoc_id = SCTP_ALL_ASSOC; srs->srs_flags = SCTP_STREAM_RESET_OUTGOING; srs->srs_number_streams = 1; srs->srs_stream_list[0] = stream_id; - usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, SCTP_RESET_STREAMS, - srs, length); + srs->srs_assoc_id = self->sctp_assoc_id; + if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, + SCTP_RESET_STREAMS, srs, length) < 0) { + GST_INFO_OBJECT (self, "Resetting stream id=%u failed", stream_id); + } g_free (srs); } From 31fe44140edc37f2e7eef5522c158023bc6bfa0d Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 25 Aug 2021 22:11:53 +0100 Subject: [PATCH 175/339] SCTP: require lock to be held before changing state The state change function will release the lock before emitting the property change event and relock afterwards. This simplifies the flow of calling code dramatically. Co-authored-by: John-Mark Bell --- .../ext/sctp/sctpassociation.c | 55 ++++++++----------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 6154fc73ca2..35132e8f219 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -129,6 +129,8 @@ static void handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid); static void maybe_set_state_to_ready (GstSctpAssociation * self); +static void gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, + GstSctpAssociationState new_state); static gboolean gst_sctp_association_change_state (GstSctpAssociation * self, GstSctpAssociationState new_state, gboolean lock); @@ -325,21 +327,13 @@ gst_sctp_association_set_property (GObject * object, guint prop_id, static void maybe_set_state_to_ready (GstSctpAssociation * self) { - gboolean signal_ready_state = FALSE; - g_mutex_lock (&self->association_mutex); if ((self->state == GST_SCTP_ASSOCIATION_STATE_NEW) && (self->local_port != 0 && self->remote_port != 0) && (self->packet_out_cb != NULL) && (self->packet_received_cb != NULL)) { - signal_ready_state = - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_READY, FALSE); + gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_READY); } g_mutex_unlock (&self->association_mutex); - - if (signal_ready_state) - g_object_notify_by_pspec (G_OBJECT (self), properties[PROP_STATE]); - } static void @@ -901,17 +895,13 @@ static void handle_association_changed (GstSctpAssociation * self, const struct sctp_assoc_change *sac) { - gboolean change_state = FALSE; - GstSctpAssociationState new_state; - switch (sac->sac_state) { case SCTP_COMM_UP: GST_DEBUG_OBJECT (self, "SCTP_COMM_UP"); g_mutex_lock (&self->association_mutex); if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { self->sctp_assoc_id = sac->sac_assoc_id; - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_CONNECTED; + gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_CONNECTED); GST_DEBUG_OBJECT (self, "SCTP association connected!"); } else if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_FIXME_OBJECT (self, "SCTP association already open"); @@ -925,14 +915,10 @@ handle_association_changed (GstSctpAssociation * self, /* TODO: Tear down association */ g_mutex_lock (&self->association_mutex); if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - g_mutex_unlock (&self->association_mutex); - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTING, TRUE); - g_mutex_lock (&self->association_mutex); + gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); } if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_DISCONNECTED; + gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); GST_INFO_OBJECT (self, "SCTP association disconnected!"); } g_mutex_unlock (&self->association_mutex); @@ -945,27 +931,19 @@ handle_association_changed (GstSctpAssociation * self, GST_INFO_OBJECT (self, "SCTP event SCTP_SHUTDOWN_COMP received"); g_mutex_lock (&self->association_mutex); if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - g_mutex_unlock (&self->association_mutex); - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTING, TRUE); - g_mutex_lock (&self->association_mutex); + gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); } if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_DISCONNECTED; + gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); GST_INFO_OBJECT (self, "SCTP association disconnected!"); } g_mutex_unlock (&self->association_mutex); break; case SCTP_CANT_STR_ASSOC: GST_WARNING_OBJECT (self, "SCTP event SCTP_CANT_STR_ASSOC received"); - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_ERROR; + gst_sctp_association_change_state (self, GST_SCTP_ASSOCIATION_STATE_ERROR, TRUE); break; } - - if (change_state) - gst_sctp_association_change_state (self, new_state, TRUE); } static void @@ -1005,6 +983,21 @@ handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, g_mutex_unlock (&self->association_mutex); } +static void +gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, + GstSctpAssociationState new_state) + { + /* Association mutex is held on entry */ + gboolean notify = gst_sctp_association_change_state (self, new_state, FALSE); + if (notify) { + /* Unlock the mutex to emit the property change event to avoid deadlock + * if the client calls back into this object from its event handler. */ + g_mutex_unlock (&self->association_mutex); + g_object_notify_by_pspec (G_OBJECT (self), properties[PROP_STATE]); + g_mutex_lock (&self->association_mutex); + } +} + /* Returns TRUE if lock==FALSE and notification is needed later. * Takes the mutex shortly if lock==TRUE! */ static gboolean From 9fc44af6b471a67afdcda3f567404d5f9e957b2c Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Fri, 16 Aug 2019 18:07:52 +0100 Subject: [PATCH 176/339] SCTP: only reset streams in connected state --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 35132e8f219..42324fc2146 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -565,9 +565,11 @@ gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id) srs->srs_stream_list[0] = stream_id; srs->srs_assoc_id = self->sctp_assoc_id; - if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, - SCTP_RESET_STREAMS, srs, length) < 0) { - GST_INFO_OBJECT (self, "Resetting stream id=%u failed", stream_id); + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, + SCTP_RESET_STREAMS, srs, length) < 0) { + GST_INFO_OBJECT (self, "Resetting stream id=%u failed", stream_id); + } } g_free (srs); From aec304187716cce8122463f06ba3bbf37bd6c256 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Fri, 16 Aug 2019 18:15:13 +0100 Subject: [PATCH 177/339] SCTP: factor out association changed event handlers --- .../ext/sctp/sctpassociation.c | 81 +++++++++++-------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 42324fc2146..3eaf446af8d 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -893,53 +893,68 @@ handle_notification (GstSctpAssociation * self, } } +static void +handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, + const struct sctp_assoc_change * sac) +{ + GST_INFO_OBJECT (self, "SCTP event %s received", + sac->sac_state == SCTP_COMM_LOST ? + "SCTP_COMM_LOST" : "SCTP_SHUTDOWN_COMP"); + + g_mutex_lock (&self->association_mutex); + + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); + } + + /* Fall through to ensure the transition to disconnected occurs */ + + if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); + GST_INFO_OBJECT (self, "SCTP association disconnected!"); + } + + g_mutex_unlock (&self->association_mutex); +} + +static void +handle_sctp_comm_up (GstSctpAssociation * self, + const struct sctp_assoc_change * sac) +{ + GST_INFO_OBJECT (self, "SCTP_COMM_UP"); + g_mutex_lock (&self->association_mutex); + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { + self->sctp_assoc_id = sac->sac_assoc_id; + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_CONNECTED); + GST_INFO_OBJECT (self, "SCTP association connected!"); + } else if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + GST_INFO_OBJECT (self, "SCTP association already open"); + } else { + GST_WARNING_OBJECT (self, "SCTP association in unexpected state"); + } + g_mutex_unlock (&self->association_mutex); +} + static void handle_association_changed (GstSctpAssociation * self, const struct sctp_assoc_change *sac) { switch (sac->sac_state) { case SCTP_COMM_UP: - GST_DEBUG_OBJECT (self, "SCTP_COMM_UP"); - g_mutex_lock (&self->association_mutex); - if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { - self->sctp_assoc_id = sac->sac_assoc_id; - gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_CONNECTED); - GST_DEBUG_OBJECT (self, "SCTP association connected!"); - } else if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - GST_FIXME_OBJECT (self, "SCTP association already open"); - } else { - GST_WARNING_OBJECT (self, "SCTP association in unexpected state"); - } - g_mutex_unlock (&self->association_mutex); + handle_sctp_comm_up (self, sac); break; case SCTP_COMM_LOST: - GST_WARNING_OBJECT (self, "SCTP event SCTP_COMM_LOST received"); - /* TODO: Tear down association */ - g_mutex_lock (&self->association_mutex); - if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); - } - if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); - GST_INFO_OBJECT (self, "SCTP association disconnected!"); - } - g_mutex_unlock (&self->association_mutex); + handle_sctp_comm_lost_or_shutdown (self, sac); break; case SCTP_RESTART: GST_DEBUG_OBJECT (self, "SCTP event SCTP_RESTART received"); break; case SCTP_SHUTDOWN_COMP: /* Occurs if in TCP mode when the far end sends SHUTDOWN */ - GST_INFO_OBJECT (self, "SCTP event SCTP_SHUTDOWN_COMP received"); - g_mutex_lock (&self->association_mutex); - if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); - } - if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); - GST_INFO_OBJECT (self, "SCTP association disconnected!"); - } - g_mutex_unlock (&self->association_mutex); + handle_sctp_comm_lost_or_shutdown (self, sac); break; case SCTP_CANT_STR_ASSOC: GST_WARNING_OBJECT (self, "SCTP event SCTP_CANT_STR_ASSOC received"); From c67e80b80ace2994c39d5dc10f04d5fcb4a1cc98 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Thu, 26 Aug 2021 11:41:38 +0100 Subject: [PATCH 178/339] SCTP: reset association when disconnected Co-authored-by: John-Mark Bell --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 3eaf446af8d..a782b3d4143 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -109,6 +109,7 @@ static void gst_sctp_association_set_property (GObject * object, guint prop_id, static void gst_sctp_association_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static void force_close (GstSctpAssociation * self, gboolean change_state); static struct socket *create_sctp_socket (GstSctpAssociation * gst_sctp_association); static struct sockaddr_conn get_sctp_socket_address (GstSctpAssociation * @@ -577,6 +578,12 @@ gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id) void gst_sctp_association_force_close (GstSctpAssociation * self) +{ + force_close(self, TRUE); +} + +static void +force_close (GstSctpAssociation * self, gboolean change_state) { if (self->sctp_ass_sock) { struct socket *s = self->sctp_ass_sock; @@ -586,8 +593,10 @@ gst_sctp_association_force_close (GstSctpAssociation * self) self->sctp_assoc_id = 0; - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTED, TRUE); + if (change_state) { + gst_sctp_association_change_state (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTED, TRUE); + } } static struct socket * @@ -911,6 +920,7 @@ handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, /* Fall through to ensure the transition to disconnected occurs */ if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { + force_close(self, FALSE); gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); GST_INFO_OBJECT (self, "SCTP association disconnected!"); From 2781accefc30c685ed7c479a8beef8bb846a88fb Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Fri, 16 Aug 2019 19:09:34 +0100 Subject: [PATCH 179/339] SCTP: permit starting association when disconnected --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index a782b3d4143..6c4c1c54f8c 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -404,7 +404,8 @@ gst_sctp_association_get (guint32 association_id) gboolean gst_sctp_association_start (GstSctpAssociation * self) { - if (self->state != GST_SCTP_ASSOCIATION_STATE_READY) { + if (self->state != GST_SCTP_ASSOCIATION_STATE_READY && + self->state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { GST_WARNING_OBJECT (self, "SCTP association is in wrong state and cannot be started"); goto configure_required; From f0c605aabe19e1fd91e400ae9a6e7e87ca6cb960 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Fri, 16 Aug 2019 20:30:19 +0100 Subject: [PATCH 180/339] SCTP: expose a reconnect action --- .../gst-plugins-bad/ext/sctp/gstsctpenc.c | 22 +++++++++++++++++++ .../gst-plugins-bad/ext/sctp/gstsctpenc.h | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index fbcf37450da..1a15b1696df 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -51,6 +51,7 @@ enum { SIGNAL_SCTP_ASSOCIATION_ESTABLISHED, SIGNAL_GET_STREAM_BYTES_SENT, + SIGNAL_RECONNECT, NUM_SIGNALS }; @@ -166,6 +167,7 @@ static void get_config_from_caps (const GstCaps * caps, gboolean * ordered, GstSctpAssociationPartialReliability * reliability, guint32 * reliability_param, guint32 * ppid, gboolean * ppid_available); static guint64 on_get_stream_bytes_sent (GstSctpEnc * self, guint stream_id); +static void reconnect (GstSctpEnc * self); static void gst_sctp_enc_class_init (GstSctpEncClass * klass) @@ -235,8 +237,14 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) G_STRUCT_OFFSET (GstSctpEncClass, on_get_stream_bytes_sent), NULL, NULL, NULL, G_TYPE_UINT64, 1, G_TYPE_UINT); + signals[SIGNAL_RECONNECT] = g_signal_new ("reconnect", + G_TYPE_FROM_CLASS (gobject_class), G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, + G_STRUCT_OFFSET (GstSctpEncClass, reconnect), NULL, NULL, + g_cclosure_marshal_generic, G_TYPE_NONE, 0); + klass->on_get_stream_bytes_sent = GST_DEBUG_FUNCPTR (on_get_stream_bytes_sent); + klass->reconnect = GST_DEBUG_FUNCPTR (reconnect); gst_element_class_set_static_metadata (element_class, "SCTP Encoder", @@ -844,6 +852,20 @@ gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event) return ret; } +static void +reconnect (GstSctpEnc * self) +{ + gint state; + + g_object_get (self->sctp_association, "state", &state, NULL); + if (state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { + GST_WARNING_OBJECT (self, "Cannot reconnect in non-DISCONNECTED state."); + return; + } + + gst_sctp_association_start (self->sctp_association); +} + static gboolean configure_association (GstSctpEnc * self) { diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index 71a79f9845e..ec37d0925aa 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -69,7 +69,7 @@ struct _GstSctpEncClass gboolean established); guint64 (*on_get_stream_bytes_sent) (GstSctpEnc * sctp_enc, guint stream_id); - + void (*reconnect) (GstSctpEnc * sctp_enc); }; GType gst_sctp_enc_get_type (void); From 3e8f247e1c47ce16b63009a789fb1a22fbca9d4f Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Fri, 16 Aug 2019 20:31:13 +0100 Subject: [PATCH 181/339] SCTP: emit signal when association is restarted --- .../gst-plugins-bad/ext/sctp/gstsctpdec.c | 22 +++++++++++++++++++ .../gst-plugins-bad/ext/sctp/gstsctpdec.h | 2 ++ .../ext/sctp/sctpassociation.c | 10 ++++++++- .../ext/sctp/sctpassociation.h | 1 + 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index a90f89428f2..3817c381988 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -53,6 +53,7 @@ GST_STATIC_PAD_TEMPLATE ("src_%u", GST_PAD_SRC, enum { SIGNAL_RESET_STREAM, + SIGNAL_ASSOC_RESTART, NUM_SIGNALS }; @@ -157,6 +158,8 @@ static void gst_sctp_data_srcpad_loop (GstPad * pad); static gboolean configure_association (GstSctpDec * self); static void on_gst_sctp_association_stream_reset (GstSctpAssociation * gst_sctp_association, guint16 stream_id, GstSctpDec * self); +static void on_gst_sctp_association_restart (GstSctpAssociation * + gst_sctp_association, GstSctpDec * self); static void on_receive (GstSctpAssociation * gst_sctp_association, guint8 * buf, gsize length, guint16 stream_id, guint ppid, gpointer user_data); @@ -215,6 +218,11 @@ gst_sctp_dec_class_init (GstSctpDecClass * klass) G_STRUCT_OFFSET (GstSctpDecClass, on_reset_stream), NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_UINT); + signals[SIGNAL_ASSOC_RESTART] = g_signal_new ("sctp-association-restarted", + G_TYPE_FROM_CLASS (gobject_class), G_SIGNAL_RUN_LAST, + G_STRUCT_OFFSET (GstSctpDecClass, on_association_restart), NULL, NULL, + g_cclosure_marshal_generic, G_TYPE_NONE, 0); + gst_element_class_set_static_metadata (element_class, "SCTP Decoder", "Decoder/Network/SCTP", @@ -484,6 +492,10 @@ configure_association (GstSctpDec * self) g_signal_connect_object (self->sctp_association, "stream-reset", G_CALLBACK (on_gst_sctp_association_stream_reset), self, 0); + self->signal_handler_association_restart = + g_signal_connect_object (self->sctp_association, "association-restart", + G_CALLBACK (on_gst_sctp_association_restart), self, 0); + g_object_bind_property (self, "local-sctp-port", self->sctp_association, "local-port", G_BINDING_SYNC_CREATE); @@ -642,6 +654,14 @@ on_gst_sctp_association_stream_reset (GstSctpAssociation * gst_sctp_association, gst_object_unref (srcpad); } +static void +on_gst_sctp_association_restart (GstSctpAssociation * gst_sctp_association, + GstSctpDec * self) +{ + (void ) gst_sctp_association; + g_signal_emit (self, signals[SIGNAL_ASSOC_RESTART], 0); +} + static void data_queue_item_free (GstDataQueueItem * item) { @@ -722,6 +742,8 @@ sctpdec_cleanup (GstSctpDec * self) if (self->sctp_association) { gst_sctp_association_set_on_packet_received (self->sctp_association, NULL, NULL, NULL); + g_signal_handler_disconnect (self->sctp_association, + self->signal_handler_association_restart); g_signal_handler_disconnect (self->sctp_association, self->signal_handler_stream_reset); gst_sctp_association_force_close (self->sctp_association); diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h index c6c89865718..43269b2629a 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h @@ -53,6 +53,7 @@ struct _GstSctpDec GstSctpAssociation *sctp_association; gulong signal_handler_stream_reset; + gulong signal_handler_association_restart; }; struct _GstSctpDecClass @@ -60,6 +61,7 @@ struct _GstSctpDecClass GstElementClass parent_class; void (*on_reset_stream) (GstSctpDec * sctp_dec, guint stream_id); + void (*on_association_restart) (GstSctpDec * sctp_dec); }; GType gst_sctp_dec_get_type (void); diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 6c4c1c54f8c..d7367466b56 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -72,6 +72,7 @@ G_DEFINE_TYPE (GstSctpAssociation, gst_sctp_association, G_TYPE_OBJECT); enum { SIGNAL_STREAM_RESET, + SIGNAL_ASSOC_RESTART, LAST_SIGNAL }; @@ -151,6 +152,12 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) G_SIGNAL_RUN_FIRST, G_STRUCT_OFFSET (GstSctpAssociationClass, on_sctp_stream_reset), NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_UINT); + signals[SIGNAL_ASSOC_RESTART] = + g_signal_new ("association-restart", G_OBJECT_CLASS_TYPE (klass), + G_SIGNAL_RUN_FIRST, G_STRUCT_OFFSET (GstSctpAssociationClass, + on_sctp_association_restart), NULL, NULL, g_cclosure_marshal_generic, + G_TYPE_NONE, 0); + properties[PROP_ASSOCIATION_ID] = g_param_spec_uint ("association-id", "The SCTP association-id", "The SCTP association-id.", 0, G_MAXUSHORT, DEFAULT_LOCAL_SCTP_PORT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); @@ -961,7 +968,8 @@ handle_association_changed (GstSctpAssociation * self, handle_sctp_comm_lost_or_shutdown (self, sac); break; case SCTP_RESTART: - GST_DEBUG_OBJECT (self, "SCTP event SCTP_RESTART received"); + GST_INFO_OBJECT (self, "SCTP event SCTP_RESTART received"); + g_signal_emit (self, signals[SIGNAL_ASSOC_RESTART], 0); break; case SCTP_SHUTDOWN_COMP: /* Occurs if in TCP mode when the far end sends SHUTDOWN */ diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index ddc6073fb27..8a2789d9557 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -99,6 +99,7 @@ struct _GstSctpAssociationClass void (*on_sctp_stream_reset) (GstSctpAssociation * sctp_association, guint16 stream_id); + void (*on_sctp_association_restart) (GstSctpAssociation * sctp_association); }; GType gst_sctp_association_get_type (void); From 2e62974e119a4321f72914d5946c694322800b4b Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Sat, 17 Aug 2019 02:14:09 +0100 Subject: [PATCH 182/339] SCTP: apply aggressive heartbeat using setsockopt Additionally, reduce the numbers. --- .../ext/sctp/sctpassociation.c | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index d7367466b56..180b58dd608 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -184,8 +184,8 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) properties[PROP_AGGRESSIVE_HEARTBEAT] = g_param_spec_boolean ("aggressive-heartbeat", "Aggressive heartbeat", - "When set to TRUE, set the heartbeat interval to 1000ms and the assoc " - "rtx max to 2.", + "When set to TRUE, set the heartbeat interval to 10ms and the assoc " + "rtx max to 1.", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); g_object_class_install_properties (gobject_class, NUM_PROPERTIES, properties); @@ -309,13 +309,6 @@ gst_sctp_association_set_property (GObject * object, guint prop_id, break; case PROP_AGGRESSIVE_HEARTBEAT: self->aggressive_heartbeat = g_value_get_boolean (value); - if (self->aggressive_heartbeat) { - usrsctp_sysctl_set_sctp_heartbeat_interval_default (1000); - usrsctp_sysctl_set_sctp_assoc_rtx_max_default (2); - } else { - usrsctp_sysctl_set_sctp_heartbeat_interval_default (30000); - usrsctp_sysctl_set_sctp_assoc_rtx_max_default (10); - } break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); @@ -937,6 +930,35 @@ handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, g_mutex_unlock (&self->association_mutex); } +static void +_apply_aggressive_heartbeat_unlocked (GstSctpAssociation * self) +{ + struct sctp_assocparams assoc_params; + struct sctp_paddrparams peer_addr_params; + struct sockaddr_conn addr; + + if (!self->aggressive_heartbeat) + return; + + memset (&assoc_params, 0, sizeof (assoc_params)); + assoc_params.sasoc_assoc_id = self->sctp_assoc_id; + assoc_params.sasoc_asocmaxrxt = 1; + if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, + SCTP_ASSOCINFO, &assoc_params, sizeof (assoc_params))) { + GST_WARNING_OBJECT (self, "Could not set SCTP_ASSOCINFO"); + } + + addr = get_sctp_socket_address (self, self->remote_port); + memset (&peer_addr_params, 0, sizeof (peer_addr_params)); + memcpy (&peer_addr_params.spp_address, &addr, sizeof (addr)); + peer_addr_params.spp_flags = SPP_HB_ENABLE; + peer_addr_params.spp_hbinterval = 10; + if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, + SCTP_PEER_ADDR_PARAMS, &peer_addr_params, sizeof (peer_addr_params))) { + GST_WARNING_OBJECT (self, "Could not set SCTP_PEER_ADDR_PARAMS"); + } +} + static void handle_sctp_comm_up (GstSctpAssociation * self, const struct sctp_assoc_change * sac) @@ -945,6 +967,7 @@ handle_sctp_comm_up (GstSctpAssociation * self, g_mutex_lock (&self->association_mutex); if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { self->sctp_assoc_id = sac->sac_assoc_id; + _apply_aggressive_heartbeat_unlocked (self); gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_CONNECTED); GST_INFO_OBJECT (self, "SCTP association connected!"); From aeb6f6a8ae8a0ea321b645fec69ef5b22ea7ed41 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Sat, 17 Aug 2019 02:38:46 +0100 Subject: [PATCH 183/339] SCTP: handle reconnect race --- subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c | 12 +++++++++++- subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 1a15b1696df..2ab41167f4b 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -858,6 +858,11 @@ reconnect (GstSctpEnc * self) gint state; g_object_get (self->sctp_association, "state", &state, NULL); + if (state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { + self->reconnect_pending = TRUE; + return; + } + if (state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { GST_WARNING_OBJECT (self, "Cannot reconnect in non-DISCONNECTED state."); return; @@ -927,10 +932,15 @@ on_sctp_association_state_changed (GstSctpAssociation * sctp_association, TRUE); break; case GST_SCTP_ASSOCIATION_STATE_DISCONNECTING: - case GST_SCTP_ASSOCIATION_STATE_DISCONNECTED: g_signal_emit (self, signals[SIGNAL_SCTP_ASSOCIATION_ESTABLISHED], 0, FALSE); break; + case GST_SCTP_ASSOCIATION_STATE_DISCONNECTED: + if (self->reconnect_pending) { + self->reconnect_pending = FALSE; + reconnect (self); + } + break; case GST_SCTP_ASSOCIATION_STATE_ERROR: GST_ELEMENT_ERROR (self, RESOURCE, WRITE, (NULL), ("SCTP association went into error state")); diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index ec37d0925aa..e0661d2a8f3 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -52,6 +52,7 @@ struct _GstSctpEnc guint16 remote_sctp_port; gboolean use_sock_stream; gboolean aggressive_heartbeat; + gboolean reconnect_pending; GstSctpAssociation *sctp_association; GstDataQueue *outbound_sctp_packet_queue; From 1582ba19cfc4d653496346308884e1a893dcdcb0 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Sat, 17 Aug 2019 12:39:09 +0100 Subject: [PATCH 184/339] SCTP: add action for explicit disconnection --- .../gst-plugins-bad/ext/sctp/gstsctpenc.c | 23 ++++++++++ .../gst-plugins-bad/ext/sctp/gstsctpenc.h | 1 + .../ext/sctp/sctpassociation.c | 42 ++++++++++++------- 3 files changed, 51 insertions(+), 15 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 2ab41167f4b..001ec891354 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -51,6 +51,7 @@ enum { SIGNAL_SCTP_ASSOCIATION_ESTABLISHED, SIGNAL_GET_STREAM_BYTES_SENT, + SIGNAL_DISCONNECT, SIGNAL_RECONNECT, NUM_SIGNALS }; @@ -167,6 +168,7 @@ static void get_config_from_caps (const GstCaps * caps, gboolean * ordered, GstSctpAssociationPartialReliability * reliability, guint32 * reliability_param, guint32 * ppid, gboolean * ppid_available); static guint64 on_get_stream_bytes_sent (GstSctpEnc * self, guint stream_id); +static gboolean disconnect (GstSctpEnc * self); static void reconnect (GstSctpEnc * self); static void @@ -237,6 +239,11 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) G_STRUCT_OFFSET (GstSctpEncClass, on_get_stream_bytes_sent), NULL, NULL, NULL, G_TYPE_UINT64, 1, G_TYPE_UINT); + signals[SIGNAL_DISCONNECT] = g_signal_new ("disconnect", + G_TYPE_FROM_CLASS (gobject_class), G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, + G_STRUCT_OFFSET (GstSctpEncClass, disconnect), NULL, NULL, + g_cclosure_marshal_generic, G_TYPE_BOOLEAN, 0); + signals[SIGNAL_RECONNECT] = g_signal_new ("reconnect", G_TYPE_FROM_CLASS (gobject_class), G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (GstSctpEncClass, reconnect), NULL, NULL, @@ -244,6 +251,7 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) klass->on_get_stream_bytes_sent = GST_DEBUG_FUNCPTR (on_get_stream_bytes_sent); + klass->disconnect = GST_DEBUG_FUNCPTR (disconnect); klass->reconnect = GST_DEBUG_FUNCPTR (reconnect); gst_element_class_set_static_metadata (element_class, @@ -852,6 +860,21 @@ gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event) return ret; } +static gboolean +disconnect (GstSctpEnc * self) +{ + gint state; + + g_object_get (self->sctp_association, "state", &state, NULL); + if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + GST_WARNING_OBJECT (self, "Cannot disconnect in non-CONNECTED state."); + return FALSE; + } + + gst_sctp_association_disconnect (self->sctp_association); + return TRUE; +} + static void reconnect (GstSctpEnc * self) { diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index e0661d2a8f3..7905af76dc8 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -70,6 +70,7 @@ struct _GstSctpEncClass gboolean established); guint64 (*on_get_stream_bytes_sent) (GstSctpEnc * sctp_enc, guint stream_id); + gboolean (*disconnect) (GstSctpEnc * sctp_enc); void (*reconnect) (GstSctpEnc * sctp_enc); }; diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 180b58dd608..62976f5e097 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -600,6 +600,32 @@ force_close (GstSctpAssociation * self, gboolean change_state) } } +static void +gst_sctp_association_disconnect_unlocked (GstSctpAssociation * self) +{ + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); + } + + /* Fall through to ensure the transition to disconnected occurs */ + + if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { + force_close(self, FALSE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); + GST_INFO_OBJECT (self, "SCTP association disconnected!"); + } +} + +void +gst_sctp_association_disconnect (GstSctpAssociation * self) +{ + g_mutex_lock (&self->association_mutex); + gst_sctp_association_disconnect_unlocked (self); + g_mutex_unlock (&self->association_mutex); +} + static struct socket * create_sctp_socket (GstSctpAssociation * self) { @@ -912,21 +938,7 @@ handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, "SCTP_COMM_LOST" : "SCTP_SHUTDOWN_COMP"); g_mutex_lock (&self->association_mutex); - - if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - gst_sctp_association_change_state_unlocked (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); - } - - /* Fall through to ensure the transition to disconnected occurs */ - - if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - force_close(self, FALSE); - gst_sctp_association_change_state_unlocked (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); - GST_INFO_OBJECT (self, "SCTP association disconnected!"); - } - + gst_sctp_association_disconnect_unlocked (self); g_mutex_unlock (&self->association_mutex); } From 0e7489bdcc3c0c187a3ae4b87af664e9a4eee4fe Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Sun, 18 Aug 2019 13:24:01 +0100 Subject: [PATCH 185/339] SCTP: move handle_sctp_comm_lost_or_shutdown --- .../ext/sctp/sctpassociation.c | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 62976f5e097..a4e6e3229bd 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -929,19 +929,6 @@ handle_notification (GstSctpAssociation * self, } } -static void -handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, - const struct sctp_assoc_change * sac) -{ - GST_INFO_OBJECT (self, "SCTP event %s received", - sac->sac_state == SCTP_COMM_LOST ? - "SCTP_COMM_LOST" : "SCTP_SHUTDOWN_COMP"); - - g_mutex_lock (&self->association_mutex); - gst_sctp_association_disconnect_unlocked (self); - g_mutex_unlock (&self->association_mutex); -} - static void _apply_aggressive_heartbeat_unlocked (GstSctpAssociation * self) { @@ -991,6 +978,19 @@ handle_sctp_comm_up (GstSctpAssociation * self, g_mutex_unlock (&self->association_mutex); } +static void +handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, + const struct sctp_assoc_change * sac) +{ + GST_INFO_OBJECT (self, "SCTP event %s received", + sac->sac_state == SCTP_COMM_LOST ? + "SCTP_COMM_LOST" : "SCTP_SHUTDOWN_COMP"); + + g_mutex_lock (&self->association_mutex); + gst_sctp_association_disconnect_unlocked (self); + g_mutex_unlock (&self->association_mutex); +} + static void handle_association_changed (GstSctpAssociation * self, const struct sctp_assoc_change *sac) From 2433d88a00a667ea62117ce92b51cfc1d4364b74 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Sun, 18 Aug 2019 13:35:51 +0100 Subject: [PATCH 186/339] SCTP: shutdown stream sockets --- .../ext/sctp/sctpassociation.c | 26 +++++++++++++++---- .../ext/sctp/sctpassociation.h | 1 + 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index a4e6e3229bd..d02d8a0d49f 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -601,11 +601,26 @@ force_close (GstSctpAssociation * self, gboolean change_state) } static void -gst_sctp_association_disconnect_unlocked (GstSctpAssociation * self) +gst_sctp_association_disconnect_unlocked (GstSctpAssociation * self, + gboolean try_shutdown) { if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); + + if (try_shutdown && self->use_sock_stream && self->sctp_ass_sock) { + GST_INFO_OBJECT (self, "SCTP association shutting down"); + self->shutdown = FALSE; + if (usrsctp_shutdown (self->sctp_ass_sock, SHUT_RDWR) == 0) { + /* wait for shutdown to complete */ + guint cs_to_wait = 100; /* 1s */ + while (!self->shutdown && cs_to_wait > 0) { + g_usleep (G_USEC_PER_SEC / 100); + cs_to_wait--; + } + self->shutdown = FALSE; + } + } } /* Fall through to ensure the transition to disconnected occurs */ @@ -622,7 +637,7 @@ void gst_sctp_association_disconnect (GstSctpAssociation * self) { g_mutex_lock (&self->association_mutex); - gst_sctp_association_disconnect_unlocked (self); + gst_sctp_association_disconnect_unlocked (self, TRUE); g_mutex_unlock (&self->association_mutex); } @@ -848,8 +863,9 @@ receive_cb (struct socket *sock, union sctp_sockstore addr, void *data, GstSctpAssociation *self = GST_SCTP_ASSOCIATION (ulp_info); if (!data) { - /* Not sure if this can happend. */ - GST_WARNING_OBJECT (self, "Received empty data buffer"); + /* This is a notification that socket shutdown is complete */ + GST_INFO_OBJECT (self, "Received shutdown complete notification"); + self->shutdown = TRUE; } else { if (flags & MSG_NOTIFICATION) { handle_notification (self, (const union sctp_notification *) data, @@ -987,7 +1003,7 @@ handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, "SCTP_COMM_LOST" : "SCTP_SHUTDOWN_COMP"); g_mutex_lock (&self->association_mutex); - gst_sctp_association_disconnect_unlocked (self); + gst_sctp_association_disconnect_unlocked (self, FALSE); g_mutex_unlock (&self->association_mutex); } diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 8a2789d9557..94cfeb0b18a 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -77,6 +77,7 @@ struct _GstSctpAssociation gboolean use_sock_stream; gboolean aggressive_heartbeat; struct socket *sctp_ass_sock; + gboolean shutdown; GMutex association_mutex; From 61e962fef22521cbcd296e67b034d14a9e2917e9 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Wed, 18 Sep 2019 20:43:44 +0100 Subject: [PATCH 187/339] SCTP: delay sctp-association-established(FALSE) --- subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c | 13 ++----------- subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h | 1 - 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 001ec891354..836231e3d7e 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -881,11 +881,6 @@ reconnect (GstSctpEnc * self) gint state; g_object_get (self->sctp_association, "state", &state, NULL); - if (state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - self->reconnect_pending = TRUE; - return; - } - if (state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { GST_WARNING_OBJECT (self, "Cannot reconnect in non-DISCONNECTED state."); return; @@ -955,14 +950,10 @@ on_sctp_association_state_changed (GstSctpAssociation * sctp_association, TRUE); break; case GST_SCTP_ASSOCIATION_STATE_DISCONNECTING: - g_signal_emit (self, signals[SIGNAL_SCTP_ASSOCIATION_ESTABLISHED], 0, - FALSE); break; case GST_SCTP_ASSOCIATION_STATE_DISCONNECTED: - if (self->reconnect_pending) { - self->reconnect_pending = FALSE; - reconnect (self); - } + g_signal_emit (self, signals[SIGNAL_SCTP_ASSOCIATION_ESTABLISHED], 0, + FALSE); break; case GST_SCTP_ASSOCIATION_STATE_ERROR: GST_ELEMENT_ERROR (self, RESOURCE, WRITE, (NULL), diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index 7905af76dc8..35a2ce3a0cd 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -52,7 +52,6 @@ struct _GstSctpEnc guint16 remote_sctp_port; gboolean use_sock_stream; gboolean aggressive_heartbeat; - gboolean reconnect_pending; GstSctpAssociation *sctp_association; GstDataQueue *outbound_sctp_packet_queue; From bc3fca3da5d938b7451441293c20d67ea8e441a0 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Fri, 20 Sep 2019 00:34:32 +0100 Subject: [PATCH 188/339] SCTP: isolate sticky events on sctpdec src pads --- .../gst-plugins-bad/ext/sctp/gstsctpdec.c | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index 3817c381988..1a93c599f89 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -383,8 +383,9 @@ gst_sctp_dec_packet_event (GstPad * pad, GstSctpDec * self, GstEvent * event) switch (GST_EVENT_TYPE (event)) { case GST_EVENT_STREAM_START: case GST_EVENT_CAPS: - /* We create our own stream-start events and the caps event does not - * make sense */ + case GST_EVENT_SEGMENT: + /* We create our own stream-start and segment events and the + * caps event does not make sense */ gst_event_unref (event); return TRUE; case GST_EVENT_EOS: @@ -535,16 +536,29 @@ gst_sctp_dec_src_event (GstPad * pad, GstSctpDec * self, GstEvent * event) } } -static gboolean -copy_sticky_events (GstPad * pad, GstEvent ** event, gpointer user_data) +static void +send_sticky_events (GstSctpDec * self, GstPad * pad, guint16 stream_id) { - GstPad *new_pad = user_data; + GstSegment segment; + gchar *pad_stream_id; + gboolean ret; - if (GST_EVENT_TYPE (*event) != GST_EVENT_CAPS - && GST_EVENT_TYPE (*event) != GST_EVENT_STREAM_START) - gst_pad_store_sticky_event (new_pad, *event); + pad_stream_id = + gst_pad_create_stream_id_printf (pad, GST_ELEMENT (self), "%hu", + stream_id); + ret = gst_pad_push_event (pad, gst_event_new_stream_start (pad_stream_id)); + g_free (pad_stream_id); + if (ret == FALSE) { + GST_ERROR_OBJECT (self, + "Pushing stream-start event failed on pad %" GST_PTR_FORMAT, pad); + } - return TRUE; + gst_segment_init (&segment, GST_FORMAT_TIME); + ret = gst_pad_push_event (pad, gst_event_new_segment (&segment)); + if (ret == FALSE) { + GST_ERROR_OBJECT (self, + "Pushing segment event failed on pad %" GST_PTR_FORMAT, pad); + } } static GstPad * @@ -552,7 +566,7 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) { GstPad *new_pad = NULL; gint state; - gchar *pad_name, *pad_stream_id; + gchar *pad_name; GstPadTemplate *template; pad_name = g_strdup_printf ("src_%hu", stream_id); @@ -587,12 +601,7 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) if (!gst_pad_set_active (new_pad, TRUE)) goto error_cleanup; - pad_stream_id = - gst_pad_create_stream_id_printf (new_pad, GST_ELEMENT (self), "%hu", - stream_id); - gst_pad_push_event (new_pad, gst_event_new_stream_start (pad_stream_id)); - g_free (pad_stream_id); - gst_pad_sticky_events_foreach (self->sink_pad, copy_sticky_events, new_pad); + send_sticky_events (self, new_pad, stream_id); if (!gst_element_add_pad (GST_ELEMENT (self), new_pad)) goto error_add; From 0f16ca2799e342bf09065b0d3c20fc76eb0f4f9c Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Fri, 20 Sep 2019 00:35:14 +0100 Subject: [PATCH 189/339] SCTPDec: ignore buffers pushed after cleanup --- subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index 1a93c599f89..92bf6603585 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -344,8 +344,10 @@ gst_sctp_dec_packet_chain (GstPad * pad, GstSctpDec * self, GstBuffer * buf) return GST_FLOW_ERROR; } - gst_sctp_association_incoming_packet (self->sctp_association, - (const guint8 *) map.data, (guint32) map.size); + if (self->sctp_association != NULL) { + gst_sctp_association_incoming_packet (self->sctp_association, + (const guint8 *) map.data, (guint32) map.size); + } gst_buffer_unmap (buf, &map); gst_buffer_unref (buf); From 5f58ed91437be94701fa4fdbd8cabe8f5bbd667c Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Tue, 24 Sep 2019 15:05:47 +0100 Subject: [PATCH 190/339] SCTP: add missing declaration --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h | 1 + 1 file changed, 1 insertion(+) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 94cfeb0b18a..9afa601804f 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -121,6 +121,7 @@ GstFlowReturn gst_sctp_association_send_data (GstSctpAssociation * self, void gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id); void gst_sctp_association_force_close (GstSctpAssociation * self); +void gst_sctp_association_disconnect (GstSctpAssociation * self); G_END_DECLS From 8f87415ce67e4f8b544a8bd05657728500edc387 Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Tue, 24 Sep 2019 15:10:58 +0100 Subject: [PATCH 191/339] SCTP: track association validity --- .../ext/sctp/sctpassociation.c | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index d02d8a0d49f..75d8d0348f4 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -99,8 +99,9 @@ static GParamSpec *properties[NUM_PROPERTIES]; #define DEFAULT_LOCAL_SCTP_PORT 0 #define DEFAULT_REMOTE_SCTP_PORT 0 -static GHashTable *associations = NULL; G_LOCK_DEFINE_STATIC (associations_lock); +static GHashTable *associations_by_id = NULL; +static GHashTable *ids_by_association = NULL; static guint32 number_of_associations = 0; /* Interface implementations */ @@ -263,7 +264,8 @@ gst_sctp_association_finalize (GObject * object) G_LOCK (associations_lock); - g_hash_table_remove (associations, GUINT_TO_POINTER (self->association_id)); + g_hash_table_remove (associations_by_id, GUINT_TO_POINTER (self->association_id)); + g_hash_table_remove (ids_by_association, self); usrsctp_deregister_address ((void *) self); number_of_associations--; @@ -381,19 +383,24 @@ gst_sctp_association_get (guint32 association_id) GST_DEBUG_CATEGORY_INIT (gst_sctp_debug_category, "sctplib", 0, "debug category for messages from usrsctp"); - if (!associations) { - associations = + if (!associations_by_id) { + associations_by_id = + g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, NULL); + ids_by_association = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, NULL); } association = - g_hash_table_lookup (associations, GUINT_TO_POINTER (association_id)); + g_hash_table_lookup (associations_by_id, + GUINT_TO_POINTER (association_id)); if (!association) { association = g_object_new (GST_SCTP_TYPE_ASSOCIATION, "association-id", association_id, NULL); - g_hash_table_insert (associations, GUINT_TO_POINTER (association_id), + g_hash_table_insert (associations_by_id, GUINT_TO_POINTER (association_id), association); + g_hash_table_insert (ids_by_association, association, + GUINT_TO_POINTER (association_id)); } else { g_object_ref (association); } @@ -841,6 +848,20 @@ client_role_connect (GstSctpAssociation * self) return FALSE; } +static gboolean +association_is_valid (GstSctpAssociation * self) +{ + gboolean valid = FALSE; + + G_LOCK (associations_lock); + + valid = g_hash_table_contains (ids_by_association, self); + + G_UNLOCK (associations_lock); + + return valid; +} + static int sctp_packet_out (void *addr, void *buffer, size_t length, guint8 tos, guint8 set_df) @@ -848,7 +869,7 @@ sctp_packet_out (void *addr, void *buffer, size_t length, guint8 tos, GstSctpAssociation *self = GST_SCTP_ASSOCIATION (addr); g_mutex_lock (&self->association_mutex); - if (self->packet_out_cb) { + if (association_is_valid (self) && self->packet_out_cb) { self->packet_out_cb (self, buffer, length, self->packet_out_user_data); } g_mutex_unlock (&self->association_mutex); @@ -862,6 +883,10 @@ receive_cb (struct socket *sock, union sctp_sockstore addr, void *data, { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (ulp_info); + if (!association_is_valid (self)) { + return 1; + } + if (!data) { /* This is a notification that socket shutdown is complete */ GST_INFO_OBJECT (self, "Received shutdown complete notification"); From dd064a5e2ff3ae5d9f6bb4a6d39657cca89ff610 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Thu, 26 Aug 2021 18:38:34 +0100 Subject: [PATCH 192/339] SCTP: dispose associations Co-authored-by: John-Mark Bell --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 75d8d0348f4..55d597c913f 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -105,6 +105,7 @@ static GHashTable *ids_by_association = NULL; static guint32 number_of_associations = 0; /* Interface implementations */ +static void gst_sctp_association_dispose (GObject * object); static void gst_sctp_association_finalize (GObject * object); static void gst_sctp_association_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); @@ -144,6 +145,7 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) gobject_class = (GObjectClass *) klass; + gobject_class->dispose = gst_sctp_association_dispose; gobject_class->finalize = gst_sctp_association_finalize; gobject_class->set_property = gst_sctp_association_set_property; gobject_class->get_property = gst_sctp_association_get_property; @@ -258,7 +260,7 @@ gst_sctp_association_init (GstSctpAssociation * self) } static void -gst_sctp_association_finalize (GObject * object) +gst_sctp_association_dispose (GObject * object) { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (object); @@ -274,6 +276,18 @@ gst_sctp_association_finalize (GObject * object) } G_UNLOCK (associations_lock); + if (G_OBJECT_CLASS (gst_sctp_association_parent_class)->dispose) { + G_OBJECT_CLASS (gst_sctp_association_parent_class)->dispose (object); + } +} + +static void +gst_sctp_association_finalize (GObject * object) +{ + GstSctpAssociation *self = GST_SCTP_ASSOCIATION (object); + + g_mutex_clear (&self->association_mutex); + G_OBJECT_CLASS (gst_sctp_association_parent_class)->finalize (object); } From dd545cb09abbebdc72b189b8b646b826285c88db Mon Sep 17 00:00:00 2001 From: John-Mark Bell Date: Tue, 24 Sep 2019 15:14:25 +0100 Subject: [PATCH 193/339] SCTP: unref association in dispose --- .../gst-plugins-bad/ext/sctp/gstsctpdec.c | 15 +++++++-- .../gst-plugins-bad/ext/sctp/gstsctpenc.c | 31 ++++++++++++++----- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index 92bf6603585..ec1aa84447a 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -142,6 +142,7 @@ gst_sctp_dec_pad_init (GstSctpDecPad * self) data_queue_full_cb, data_queue_empty_cb, NULL); } +static void gst_sctp_dec_dispose (GObject * object); static void gst_sctp_dec_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_sctp_dec_get_property (GObject * object, guint prop_id, @@ -187,6 +188,7 @@ gst_sctp_dec_class_init (GstSctpDecClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&sink_template)); + gobject_class->dispose = gst_sctp_dec_dispose; gobject_class->set_property = gst_sctp_dec_set_property; gobject_class->get_property = gst_sctp_dec_get_property; gobject_class->finalize = gst_sctp_dec_finalize; @@ -247,6 +249,17 @@ gst_sctp_dec_init (GstSctpDec * self) gst_element_add_pad (GST_ELEMENT (self), self->sink_pad); } +static void +gst_sctp_dec_dispose (GObject * object) +{ + GstSctpDec *self = GST_SCTP_DEC (object); + + g_object_unref (self->sctp_association); + self->sctp_association = NULL; + + G_OBJECT_CLASS (gst_sctp_dec_parent_class)->dispose (object); +} + static void gst_sctp_dec_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) @@ -758,8 +771,6 @@ sctpdec_cleanup (GstSctpDec * self) g_signal_handler_disconnect (self->sctp_association, self->signal_handler_stream_reset); gst_sctp_association_force_close (self->sctp_association); - g_object_unref (self->sctp_association); - self->sctp_association = NULL; } } diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 836231e3d7e..c573badfee3 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -139,6 +139,7 @@ gst_sctp_enc_pad_init (GstSctpEncPad * self) self->clear_to_send = FALSE; } +static void gst_sctp_enc_dispose (GObject * object); static void gst_sctp_enc_finalize (GObject * object); static void gst_sctp_enc_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); @@ -188,6 +189,7 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), gst_static_pad_template_get (&sink_template)); + gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_sctp_enc_dispose); gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_sctp_enc_finalize); gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_sctp_enc_set_property); gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_sctp_enc_get_property); @@ -299,6 +301,19 @@ gst_sctp_enc_init (GstSctpEnc * self) self->src_ret = GST_FLOW_FLUSHING; } +static void +gst_sctp_enc_dispose (GObject * object) +{ + GstSctpEnc *self = GST_SCTP_ENC (object); + + if (self->sctp_association) { + g_object_unref (self->sctp_association); + self->sctp_association = NULL; + } + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + static void gst_sctp_enc_finalize (GObject * object) { @@ -1035,14 +1050,14 @@ sctpenc_cleanup (GstSctpEnc * self) { GstIterator *it; - gst_sctp_association_set_on_packet_out (self->sctp_association, NULL, NULL, - NULL); - - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_state_changed); - gst_sctp_association_force_close (self->sctp_association); - g_object_unref (self->sctp_association); - self->sctp_association = NULL; + if (self->sctp_association) { + gst_sctp_association_set_on_packet_out (self->sctp_association, NULL, NULL, + NULL); + g_signal_handler_disconnect (self->sctp_association, + self->signal_handler_state_changed); + stop_srcpad_task (self->src_pad, self); + gst_sctp_association_force_close (self->sctp_association); + } it = gst_element_iterate_sink_pads (GST_ELEMENT (self)); while (gst_iterator_foreach (it, remove_sinkpad, self) == GST_ITERATOR_RESYNC) From 3596f6582dfab225d77b07851d949e557e146184 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 18 Aug 2021 16:53:52 +0100 Subject: [PATCH 194/339] usrsctp: port pexip/usrsctp 'Fix deadlock during shutdown' commit a8b63ae927b6dddeeeae5d83803a0e8e1c7b2a3f Author: John Bassett Date: Tue May 21 18:23:05 2019 +0100 Fix deadlock during shutdown Deadlock was caused by one thread calling close on a socket whilst another thread was blocked on a call to recvmsg. In this scenario the recv thread would wake up after the recvmsg had timed out but then attempt another read from the now closed socket. Normally this would result in an error, but if the file descriptor had been reused in the meantime the recvmsg call could end up blocking forever trying to read from the wrong fd. - Make sure the socket fds used are reset to -1 before closing to prevent other threads attempting to read from a socket that has already been closed. - Call shutdown on sockets before closing to guarantee other threads reading from the socket are unblocked immediately. - Check for return of 0 when reading from a socket to handle sockets that have been cleanly closed from another thread. --- .../usrsctp/usrsctplib/user_recv_thread.c | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_recv_thread.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_recv_thread.c index 3c1657b4a66..3579834c817 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_recv_thread.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_recv_thread.c @@ -213,7 +213,10 @@ recv_function_route(void *arg) len = recvmsg(SCTP_BASE_VAR(userspace_route), &msg, 0); - if (len < 0) { + if (len == 0) + break; + + if (len < 0) { if (errno == EAGAIN || errno == EINTR) { continue; } else { @@ -341,6 +344,9 @@ recv_function_raw(void *arg) msg.msg_control = NULL; msg.msg_controllen = 0; ncounter = n = recvmsg(SCTP_BASE_VAR(userspace_rawsctp), &msg, 0); + if (n == 0) + break; + if (n < 0) { if (errno == EAGAIN || errno == EINTR) { continue; @@ -533,6 +539,9 @@ recv_function_raw6(void *arg) msg.msg_flags = 0; ncounter = n = recvmsg(SCTP_BASE_VAR(userspace_rawsctp6), &msg, 0); + if (n == 0) + break; + if (n < 0) { if (errno == EAGAIN || errno == EINTR) { continue; @@ -702,13 +711,18 @@ recv_function_udp(void *arg) msg.msg_flags = 0; ncounter = n = recvmsg(SCTP_BASE_VAR(userspace_udpsctp), &msg, 0); - if (n < 0) { + + if (n == 0) + break; + + if (n < 0) { if (errno == EAGAIN || errno == EINTR) { continue; } else { break; } } + #else nResult = WSAIoctl(SCTP_BASE_VAR(userspace_udpsctp), SIO_GET_EXTENSION_FUNCTION_POINTER, &WSARecvMsg_GUID, sizeof WSARecvMsg_GUID, @@ -904,6 +918,8 @@ recv_function_udp6(void *arg) msg.msg_flags = 0; ncounter = n = recvmsg(SCTP_BASE_VAR(userspace_udpsctp6), &msg, 0); + if (n == 0) + break; if (n < 0) { if (errno == EAGAIN || errno == EINTR) { continue; @@ -1452,6 +1468,19 @@ recv_thread_init(void) #endif } +#if !defined(_WIN32) +static void +close_socket(int *sock) +{ + if (*sock != -1) { + int socktmp = *sock; + *sock = -1; + shutdown (socktmp, SHUT_RDWR); + close(socktmp); + } +} +#endif /* !defined(_WIN32) */ + void recv_thread_destroy(void) { @@ -1471,7 +1500,7 @@ recv_thread_destroy(void) WaitForSingleObject(SCTP_BASE_VAR(recvthreadraw), INFINITE); CloseHandle(SCTP_BASE_VAR(recvthreadraw)); #else - close(SCTP_BASE_VAR(userspace_rawsctp)); + close_socket(&SCTP_BASE_VAR(userspace_rawsctp)); SCTP_BASE_VAR(userspace_rawsctp) = -1; pthread_join(SCTP_BASE_VAR(recvthreadraw), NULL); #endif @@ -1483,7 +1512,7 @@ recv_thread_destroy(void) WaitForSingleObject(SCTP_BASE_VAR(recvthreadudp), INFINITE); CloseHandle(SCTP_BASE_VAR(recvthreadudp)); #else - close(SCTP_BASE_VAR(userspace_udpsctp)); + close_socket(&SCTP_BASE_VAR(userspace_udpsctp)); SCTP_BASE_VAR(userspace_udpsctp) = -1; pthread_join(SCTP_BASE_VAR(recvthreadudp), NULL); #endif @@ -1497,7 +1526,7 @@ recv_thread_destroy(void) WaitForSingleObject(SCTP_BASE_VAR(recvthreadraw6), INFINITE); CloseHandle(SCTP_BASE_VAR(recvthreadraw6)); #else - close(SCTP_BASE_VAR(userspace_rawsctp6)); + close_socket(&SCTP_BASE_VAR(userspace_rawsctp6)); SCTP_BASE_VAR(userspace_rawsctp6) = -1; pthread_join(SCTP_BASE_VAR(recvthreadraw6), NULL); #endif @@ -1509,7 +1538,7 @@ recv_thread_destroy(void) WaitForSingleObject(SCTP_BASE_VAR(recvthreadudp6), INFINITE); CloseHandle(SCTP_BASE_VAR(recvthreadudp6)); #else - close(SCTP_BASE_VAR(userspace_udpsctp6)); + close_socket(&SCTP_BASE_VAR(userspace_udpsctp6)); SCTP_BASE_VAR(userspace_udpsctp6) = -1; pthread_join(SCTP_BASE_VAR(recvthreadudp6), NULL); #endif From 0ad6119ae5cd2d0a6ab038e4b320c1db929e2a17 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 18 Aug 2021 16:58:26 +0100 Subject: [PATCH 195/339] usrsctp: port pexip/usrsctp 'Forcibly cancel iterator thread' commit 3a3cb88c388919b9348b828becb8576a580eccc4 Author: John-Mark Bell Date: Thu Aug 15 22:22:59 2019 +0100 Forcibly cancel iterator thread --- .../ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c index 6fd3c5e2e1a..09afc85f1d9 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c @@ -6810,6 +6810,7 @@ sctp_pcb_finish(void) CloseHandle(sctp_it_ctl.thread_proc); sctp_it_ctl.thread_proc = NULL; #else + pthread_cancel(sctp_it_ctl.thread_proc); pthread_join(sctp_it_ctl.thread_proc, NULL); sctp_it_ctl.thread_proc = 0; #endif From 1c0d67b8dcd62ac2017aeb4222114fa01a0223ec Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 18 Aug 2021 17:06:46 +0100 Subject: [PATCH 196/339] usrsctp: port pexip/usrsctp 'Remove iterator thread' commit 7d050e2140efd73b408eecac5f4cede116b7f1f1 Author: John-Mark Bell Date: Sat Aug 17 19:15:30 2019 +0100 Remove iterator thread This is not necessary a world using AF_CONN only. Fix the one place where an iterator would be used to clean up directly in this case. --- .../sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c index 09afc85f1d9..5aaf2c09426 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c @@ -901,6 +901,15 @@ sctp_del_addr_from_vrf(uint32_t vrf_id, struct sockaddr *addr, sctp_free_ifa(sctp_ifap); return; } +#if defined(__Userspace__) + else if (sctp_ifap->address.sa.sa_family == AF_CONN) { + /* Clean up immediately */ + SCTPDBG(SCTP_DEBUG_PCB4, "Immediately deleting AF_CONN\n"); + sctp_free_ifa(sctp_ifap); + SCTP_ZONE_FREE(SCTP_BASE_INFO(ipi_zone_laddr), wi); + return; + } +#endif SCTP_INCR_LADDR_COUNT(); memset(wi, 0, sizeof(*wi)); (void)SCTP_GETTIME_TIMEVAL(&wi->start_time); @@ -6715,7 +6724,9 @@ sctp_pcb_init(void) (void)pthread_cond_init(&sctp_it_ctl.iterator_wakeup, NULL); #endif #endif +#if defined(SCTP_ITERATOR) sctp_startup_iterator(); +#endif #if defined(__FreeBSD__) && !defined(__Userspace__) #if defined(SCTP_MCORE_INPUT) && defined(SMP) @@ -6770,6 +6781,7 @@ sctp_pcb_finish(void) return; } SCTP_BASE_VAR(sctp_pcb_initialized) = 0; +#if defined(SCTP_ITERATOR) #if !(defined(__FreeBSD__) && !defined(__Userspace__)) /* Notify the iterator to exit. */ SCTP_IPI_ITERATOR_WQ_LOCK(); @@ -6816,6 +6828,7 @@ sctp_pcb_finish(void) #endif } #endif +#endif #if defined(SCTP_PROCESS_LEVEL_LOCKS) #if defined(_WIN32) DeleteConditionVariable(&sctp_it_ctl.iterator_wakeup); @@ -8063,6 +8076,10 @@ sctp_initiate_iterator(inp_func inpf, { struct sctp_iterator *it = NULL; +#if !defined(SCTP_ITERATOR) + return (-1); +#endif + if (af == NULL) { return (-1); } From fa8bcc9f3835175382ac40cf5865073df0e7689b Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 18 Aug 2021 17:25:34 +0100 Subject: [PATCH 197/339] usrsctp: port pexip/usrsctp 'init_random rename to init_usrsctp_random as conflicts with cPython bindings' commit ecf2de543c3157e507e9fc2e7c8319bc443ede39 Author: Thomas Williams Date: Thu Jun 17 14:36:26 2021 +0100 init_random rename to init_usrsctp_random as conflicts with cPython bindings --- .../ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c | 2 +- .../ext/sctp/usrsctp/usrsctplib/user_environment.c | 6 +++--- .../ext/sctp/usrsctp/usrsctplib/user_environment.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c index bf66b362006..a66c73d085b 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c @@ -87,7 +87,7 @@ sctp_init(void) u_long sb_max_adj; #else - init_random(); + init_usrsctp_random(); #endif /* Initialize and modify the sysctled variables */ sctp_init_sysctls(); diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.c index dc9d98a3920..f88cf280021 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.c @@ -66,7 +66,7 @@ userland_mutex_t atomic_mtx; */ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION void -init_random(void) +init_usrsctp_random(void) { return; } @@ -79,7 +79,7 @@ read_random(void *buf, int count) } #elif defined(__FreeBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) || defined(__OpenBSD__) || defined(__APPLE__) void -init_random(void) +init_usrsctp_random(void) { return; } @@ -94,7 +94,7 @@ read_random(void *buf, int count) } #else void -init_random(void) +init_usrsctp_random(void) { struct timeval now; unsigned int seed; diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.h b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.h index 0c2ec9db235..3414d8683b1 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.h +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.h @@ -67,7 +67,7 @@ extern int nmbclusters; #define max(a,b) (((a)>(b))?(a):(b)) #endif -void init_random(void); +void init_usrsctp_random(void); int read_random(void *, int); /* errno's may differ per OS. errno.h now included in sctp_os_userspace.h */ From 07d7841e63766757f65823fb342ecc1de7e203a3 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Thu, 16 Sep 2021 16:23:35 +0100 Subject: [PATCH 198/339] SCTP: Downgrade data on disconnect log message --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 55d597c913f..e07896c653c 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -512,7 +512,7 @@ gst_sctp_association_send_data (GstSctpAssociation * self, const guint8 * buf, if (self->state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTED || self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - GST_ERROR_OBJECT (self, "Disconnected"); + GST_INFO_OBJECT (self, "Disconnected"); flow_ret = GST_FLOW_EOS; g_mutex_unlock (&self->association_mutex); goto end; From 4afd89411c17c07d5749e57e698bc09b6469377f Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Fri, 17 Sep 2021 16:37:26 +0100 Subject: [PATCH 199/339] SCTP: retry after failure to bind with EADDRINUSE Co-authored-by: John-Mark Bell --- .../ext/sctp/sctpassociation.c | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index e07896c653c..a02e169f7a6 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -810,14 +810,31 @@ client_role_connect (GstSctpAssociation * self) remote_addr = get_sctp_socket_address (self, self->remote_port); g_mutex_unlock (&self->association_mutex); - ret = - usrsctp_bind (self->sctp_ass_sock, (struct sockaddr *) &local_addr, - sizeof (struct sockaddr_conn)); - if (ret < 0) { - GST_ERROR_OBJECT (self, "usrsctp_bind() error: (%u) %s", errno, - g_strerror (errno)); - goto error; - } + /* After an SCTP association is reported as disconnected, there is + * a window of time before the underlying SCTP stack cleans up. + * If a client-initiated reconnect request occurs during this window + * then we will attempt to bind using the same address information + * which will fail with EADDRINUSE. Handle this by retrying whenever + * a bind fails in this way. + */ + size_t retry_count = 1; + do { + ret = + usrsctp_bind (self->sctp_ass_sock, (struct sockaddr *) &local_addr, + sizeof (struct sockaddr_conn)); + if (ret < 0) { + if (errno != EADDRINUSE || retry_count == 5) { + GST_ERROR_OBJECT (self, "usrsctp_bind() error: (%u) %s", errno, + g_strerror (errno)); + goto error; + } else { + GST_WARNING_OBJECT (self, "usrsctp_bind() error: (%u) %s", errno, + g_strerror (errno)); + } + g_usleep (G_USEC_PER_SEC / 100); + retry_count++; + } + } while (ret < 0); ret = usrsctp_connect (self->sctp_ass_sock, (struct sockaddr *) &remote_addr, From 1169ac9463315b76e1b8e2b1e690647a640ac042 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 6 Oct 2021 14:59:32 +0100 Subject: [PATCH 200/339] SCTP: Fix deadlock during usrsctp_shutdown or usrsctp_close (2) --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index a02e169f7a6..e4324db79a5 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -112,7 +112,8 @@ static void gst_sctp_association_set_property (GObject * object, guint prop_id, static void gst_sctp_association_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static void force_close (GstSctpAssociation * self, gboolean change_state); +static void force_close_unlocked (GstSctpAssociation * self, + gboolean change_state); static struct socket *create_sctp_socket (GstSctpAssociation * gst_sctp_association); static struct sockaddr_conn get_sctp_socket_address (GstSctpAssociation * @@ -601,11 +602,13 @@ gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id) void gst_sctp_association_force_close (GstSctpAssociation * self) { - force_close(self, TRUE); + g_mutex_lock (&self->association_mutex); + force_close_unlocked (self, TRUE); + g_mutex_unlock (&self->association_mutex); } static void -force_close (GstSctpAssociation * self, gboolean change_state) +force_close_unlocked (GstSctpAssociation * self, gboolean change_state) { if (self->sctp_ass_sock) { struct socket *s = self->sctp_ass_sock; @@ -616,8 +619,8 @@ force_close (GstSctpAssociation * self, gboolean change_state) self->sctp_assoc_id = 0; if (change_state) { - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTED, TRUE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); } } @@ -647,7 +650,7 @@ gst_sctp_association_disconnect_unlocked (GstSctpAssociation * self, /* Fall through to ensure the transition to disconnected occurs */ if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - force_close(self, FALSE); + force_close_unlocked (self, FALSE); gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); GST_INFO_OBJECT (self, "SCTP association disconnected!"); @@ -899,11 +902,9 @@ sctp_packet_out (void *addr, void *buffer, size_t length, guint8 tos, { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (addr); - g_mutex_lock (&self->association_mutex); if (association_is_valid (self) && self->packet_out_cb) { self->packet_out_cb (self, buffer, length, self->packet_out_user_data); } - g_mutex_unlock (&self->association_mutex); return 0; } From bda403c8aadb1de47dd75cb32d64b7fc8baddbc5 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 6 Oct 2021 15:09:05 +0100 Subject: [PATCH 201/339] SCTP: disconnect-reconnect association in unexpected state --- .../ext/sctp/sctpassociation.c | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index e4324db79a5..30cc8c43421 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -424,7 +424,7 @@ gst_sctp_association_get (guint32 association_id) } gboolean -gst_sctp_association_start (GstSctpAssociation * self) +gst_sctp_association_start_unlocked (GstSctpAssociation * self) { if (self->state != GST_SCTP_ASSOCIATION_STATE_READY && self->state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { @@ -438,23 +438,32 @@ gst_sctp_association_start (GstSctpAssociation * self) /* TODO: Support both server and client role */ if (!client_role_connect (self)) { - gst_sctp_association_change_state (self, GST_SCTP_ASSOCIATION_STATE_ERROR, - TRUE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_ERROR); goto error; } - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_CONNECTING, TRUE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_CONNECTING); return TRUE; error: - gst_sctp_association_change_state (self, GST_SCTP_ASSOCIATION_STATE_ERROR, - TRUE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_ERROR); return FALSE; configure_required: return FALSE; } +gboolean +gst_sctp_association_start (GstSctpAssociation * self) +{ + g_mutex_lock (&self->association_mutex); + gboolean ret = gst_sctp_association_start_unlocked (self); + g_mutex_unlock (&self->association_mutex); + return ret; +} + void gst_sctp_association_set_on_packet_out (GstSctpAssociation * self, GstSctpAssociationPacketOutCb packet_out_cb, gpointer user_data, @@ -808,10 +817,8 @@ client_role_connect (GstSctpAssociation * self) socklen_t opt_len; gint ret; - g_mutex_lock (&self->association_mutex); local_addr = get_sctp_socket_address (self, self->local_port); remote_addr = get_sctp_socket_address (self, self->remote_port); - g_mutex_unlock (&self->association_mutex); /* After an SCTP association is reported as disconnected, there is * a window of time before the underlying SCTP stack cleans up. @@ -1046,7 +1053,7 @@ handle_sctp_comm_up (GstSctpAssociation * self, } else if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_INFO_OBJECT (self, "SCTP association already open"); } else { - GST_WARNING_OBJECT (self, "SCTP association in unexpected state"); + GST_WARNING_OBJECT (self, "SCTP association in unexpected state: %d", self->state); } g_mutex_unlock (&self->association_mutex); } From 4af7924f4b5f5bfa19772cd8c01d5c53ff468294 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Thu, 16 Dec 2021 12:41:40 +0000 Subject: [PATCH 202/339] SCTP: On EADDRINUSE increase retries for usrsctp_bind --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 30cc8c43421..bf01fe4bbe8 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -833,7 +833,7 @@ client_role_connect (GstSctpAssociation * self) usrsctp_bind (self->sctp_ass_sock, (struct sockaddr *) &local_addr, sizeof (struct sockaddr_conn)); if (ret < 0) { - if (errno != EADDRINUSE || retry_count == 5) { + if (errno != EADDRINUSE || retry_count == 10) { GST_ERROR_OBJECT (self, "usrsctp_bind() error: (%u) %s", errno, g_strerror (errno)); goto error; From fd8f95d43d0d74b18c9e194412143e5d21d69730 Mon Sep 17 00:00:00 2001 From: Will Miller Date: Tue, 21 Jun 2022 09:27:20 +0100 Subject: [PATCH 203/339] SCTPDec: don't unref NULL association The association can be NULL if the element was never started, or if the association ID was already in use. Be symmetrical with SCTPEnc and check this. --- subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index ec1aa84447a..81160b31de7 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -254,8 +254,10 @@ gst_sctp_dec_dispose (GObject * object) { GstSctpDec *self = GST_SCTP_DEC (object); - g_object_unref (self->sctp_association); - self->sctp_association = NULL; + if (self->sctp_association) { + g_object_unref (self->sctp_association); + self->sctp_association = NULL; + } G_OBJECT_CLASS (gst_sctp_dec_parent_class)->dispose (object); } From 7df38fceee67441d5da036185a1b53cd6b068213 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 3 Aug 2022 15:40:20 +0200 Subject: [PATCH 204/339] SCTP: moving the removal of the associations from dispose() to finalize() - Decrementing number_of_associations in dispose() can be troublesome, as as per documentation, the dispose function can be called many times. - Also removed extra check if dispose is present, as parent class (GObject) ensures it is available. --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index bf01fe4bbe8..2e4e17010b0 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -105,7 +105,6 @@ static GHashTable *ids_by_association = NULL; static guint32 number_of_associations = 0; /* Interface implementations */ -static void gst_sctp_association_dispose (GObject * object); static void gst_sctp_association_finalize (GObject * object); static void gst_sctp_association_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); @@ -146,7 +145,6 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) gobject_class = (GObjectClass *) klass; - gobject_class->dispose = gst_sctp_association_dispose; gobject_class->finalize = gst_sctp_association_finalize; gobject_class->set_property = gst_sctp_association_set_property; gobject_class->get_property = gst_sctp_association_get_property; @@ -261,13 +259,14 @@ gst_sctp_association_init (GstSctpAssociation * self) } static void -gst_sctp_association_dispose (GObject * object) +gst_sctp_association_finalize (GObject * object) { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (object); G_LOCK (associations_lock); - g_hash_table_remove (associations_by_id, GUINT_TO_POINTER (self->association_id)); + g_hash_table_remove (associations_by_id, + GUINT_TO_POINTER (self->association_id)); g_hash_table_remove (ids_by_association, self); usrsctp_deregister_address ((void *) self); @@ -275,17 +274,8 @@ gst_sctp_association_dispose (GObject * object) if (number_of_associations == 0) { usrsctp_finish (); } - G_UNLOCK (associations_lock); - - if (G_OBJECT_CLASS (gst_sctp_association_parent_class)->dispose) { - G_OBJECT_CLASS (gst_sctp_association_parent_class)->dispose (object); - } -} -static void -gst_sctp_association_finalize (GObject * object) -{ - GstSctpAssociation *self = GST_SCTP_ASSOCIATION (object); + G_UNLOCK (associations_lock); g_mutex_clear (&self->association_mutex); From 2728b248286eb1bdd4e46c3844c481d38d314189 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 23 Sep 2022 11:26:44 +0200 Subject: [PATCH 205/339] SCTP: fixing pthread_mutex_destroy call on a destroyed mutex The initialization of the iterators locks depended on SCTP_ITERATOR definition, but sctp_pcb_finish() calls destroy on them without that check. Note that also these locks are accessed without any check on SCTP_ITERATOR so doing the initialization here fixes that problem too. --- .../ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c index 5aaf2c09426..4c41273f187 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c @@ -6726,6 +6726,10 @@ sctp_pcb_init(void) #endif #if defined(SCTP_ITERATOR) sctp_startup_iterator(); +#else + SCTP_ITERATOR_LOCK_INIT(); + SCTP_IPI_ITERATOR_WQ_INIT(); + TAILQ_INIT(&sctp_it_ctl.iteratorhead); #endif #if defined(__FreeBSD__) && !defined(__Userspace__) From 02d44dad0ad80ee62aab9681ba79ebdb099b8041 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 27 Sep 2022 17:13:20 +0200 Subject: [PATCH 206/339] SCTP: refactoring encoder and decoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move the cast macro to do a basic cast instead of check gtype instance as that one access a global lock. - Configure and cleanup the sctp association only on state change from NULL->READY and READY->NULL. - Moved the start/stop of the srcpad task in the activate_mode function, rather than the state change. This fixes the task thread accessing invalid data. Co-Authored-By: Håvard Graff --- .../gst-plugins-bad/ext/sctp/gstsctpdec.c | 219 +++++++++--------- .../gst-plugins-bad/ext/sctp/gstsctpdec.h | 1 + .../gst-plugins-bad/ext/sctp/gstsctpenc.c | 213 +++++++++-------- .../gst-plugins-bad/ext/sctp/gstsctpenc.h | 1 + 4 files changed, 224 insertions(+), 210 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index 81160b31de7..857061a06d6 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -80,7 +80,7 @@ static GParamSpec *properties[NUM_PROPERTIES]; GType gst_sctp_dec_pad_get_type (void); #define GST_TYPE_SCTP_DEC_PAD (gst_sctp_dec_pad_get_type()) -#define GST_SCTP_DEC_PAD(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_SCTP_DEC_PAD, GstSctpDecPad)) +#define GST_SCTP_DEC_PAD_CAST(obj) (GstSctpDecPad*)(obj) #define GST_SCTP_DEC_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_DEC_PAD, GstSctpDecPadClass)) #define GST_IS_SCTP_DEC_PAD(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_SCTP_DEC_PAD)) #define GST_IS_SCTP_DEC_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_SCTP_DEC_PAD)) @@ -100,7 +100,7 @@ G_DEFINE_TYPE (GstSctpDecPad, gst_sctp_dec_pad, GST_TYPE_PAD); static void gst_sctp_dec_pad_finalize (GObject * object) { - GstSctpDecPad *self = GST_SCTP_DEC_PAD (object); + GstSctpDecPad *self = GST_SCTP_DEC_PAD_CAST (object); gst_object_unref (self->packet_queue); @@ -157,6 +157,7 @@ static gboolean gst_sctp_dec_packet_event (GstPad * pad, GstSctpDec * self, static void gst_sctp_data_srcpad_loop (GstPad * pad); static gboolean configure_association (GstSctpDec * self); +static void cleanup_association (GstSctpDec * self); static void on_gst_sctp_association_stream_reset (GstSctpAssociation * gst_sctp_association, guint16 stream_id, GstSctpDec * self); static void on_gst_sctp_association_restart (GstSctpAssociation * @@ -164,9 +165,6 @@ static void on_gst_sctp_association_restart (GstSctpAssociation * static void on_receive (GstSctpAssociation * gst_sctp_association, guint8 * buf, gsize length, guint16 stream_id, guint ppid, gpointer user_data); -static void stop_srcpad_task (GstPad * pad); -static void stop_all_srcpad_tasks (GstSctpDec * self); -static void sctpdec_cleanup (GstSctpDec * self); static GstPad *get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id); static void remove_pad (GstSctpDec * self, GstPad * pad); static void on_reset_stream (GstSctpDec * self, guint stream_id); @@ -188,9 +186,9 @@ gst_sctp_dec_class_init (GstSctpDecClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&sink_template)); - gobject_class->dispose = gst_sctp_dec_dispose; gobject_class->set_property = gst_sctp_dec_set_property; gobject_class->get_property = gst_sctp_dec_get_property; + gobject_class->dispose = gst_sctp_dec_dispose; gobject_class->finalize = gst_sctp_dec_finalize; element_class->change_state = GST_DEBUG_FUNCPTR (gst_sctp_dec_change_state); @@ -249,24 +247,61 @@ gst_sctp_dec_init (GstSctpDec * self) gst_element_add_pad (GST_ELEMENT (self), self->sink_pad); } +static void +remove_pad (GstSctpDec * self, GstPad * pad) +{ + GST_DEBUG_OBJECT (pad, "Removing pad"); + + gst_pad_set_active (pad, FALSE); + if (gst_object_has_as_parent (GST_OBJECT (pad), GST_OBJECT (self))) + gst_element_remove_pad (GST_ELEMENT (self), pad); + + GST_OBJECT_LOCK (self); + gst_flow_combiner_remove_pad (self->flow_combiner, pad); + GST_OBJECT_UNLOCK (self); +} + +static void +remove_pad_it (const GValue * item, gpointer user_data) +{ + GstPad *pad = g_value_get_object (item); + GstSctpDec *self = user_data; + + remove_pad (self, pad); +} + static void gst_sctp_dec_dispose (GObject * object) { - GstSctpDec *self = GST_SCTP_DEC (object); + GstSctpDec *self = GST_SCTP_DEC_CAST (object); + GstIterator *it; + + /* remove all srcpads */ + it = gst_element_iterate_src_pads (GST_ELEMENT (self)); + while (gst_iterator_foreach (it, remove_pad_it, self) == GST_ITERATOR_RESYNC) + gst_iterator_resync (it); + gst_iterator_free (it); - if (self->sctp_association) { - g_object_unref (self->sctp_association); - self->sctp_association = NULL; - } G_OBJECT_CLASS (gst_sctp_dec_parent_class)->dispose (object); } +static void +gst_sctp_dec_finalize (GObject * object) +{ + GstSctpDec *self = GST_SCTP_DEC_CAST (object); + + gst_flow_combiner_free (self->flow_combiner); + self->flow_combiner = NULL; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + static void gst_sctp_dec_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - GstSctpDec *self = GST_SCTP_DEC (object); + GstSctpDec *self = GST_SCTP_DEC_CAST (object); switch (prop_id) { case PROP_GST_SCTP_ASSOCIATION_ID: @@ -285,7 +320,7 @@ static void gst_sctp_dec_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - GstSctpDec *self = GST_SCTP_DEC (object); + GstSctpDec *self = GST_SCTP_DEC_CAST (object); switch (prop_id) { case PROP_GST_SCTP_ASSOCIATION_ID: @@ -300,42 +335,22 @@ gst_sctp_dec_get_property (GObject * object, guint prop_id, GValue * value, } } -static void -gst_sctp_dec_finalize (GObject * object) -{ - GstSctpDec *self = GST_SCTP_DEC (object); - - gst_flow_combiner_free (self->flow_combiner); - self->flow_combiner = NULL; - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - static GstStateChangeReturn gst_sctp_dec_change_state (GstElement * element, GstStateChange transition) { - GstSctpDec *self = GST_SCTP_DEC (element); - GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + GstSctpDec *self = GST_SCTP_DEC_CAST (element); + GstStateChangeReturn ret; + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); switch (transition) { - case GST_STATE_CHANGE_READY_TO_PAUSED: + case GST_STATE_CHANGE_NULL_TO_READY: gst_flow_combiner_reset (self->flow_combiner); if (!configure_association (self)) ret = GST_STATE_CHANGE_FAILURE; break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - stop_all_srcpad_tasks (self); - break; - default: - break; - } - - if (ret != GST_STATE_CHANGE_FAILURE) - ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); - - switch (transition) { - case GST_STATE_CHANGE_PAUSED_TO_READY: - sctpdec_cleanup (self); + case GST_STATE_CHANGE_READY_TO_NULL: + cleanup_association (self); gst_flow_combiner_reset (self->flow_combiner); break; default: @@ -389,8 +404,6 @@ flush_srcpad (const GValue * item, gpointer user_data) gst_data_queue_flush (sctpdec_pad->packet_queue); } else { gst_data_queue_set_flushing (sctpdec_pad->packet_queue, FALSE); - gst_pad_start_task (GST_PAD (sctpdec_pad), - (GstTaskFunction) gst_sctp_data_srcpad_loop, sctpdec_pad, NULL); } } @@ -440,10 +453,10 @@ static void gst_sctp_data_srcpad_loop (GstPad * pad) { GstSctpDec *self; - GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD (pad); + GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); GstDataQueueItem *item; - self = GST_SCTP_DEC (gst_pad_get_parent (pad)); + self = GST_SCTP_DEC_CAST (gst_pad_get_parent (pad)); if (gst_data_queue_pop (sctpdec_pad->packet_queue, &item)) { GstBuffer *buffer; @@ -495,7 +508,6 @@ configure_association (GstSctpDec * self) gint state; self->sctp_association = gst_sctp_association_get (self->sctp_association_id); - g_object_get (self->sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_NEW) { @@ -503,7 +515,7 @@ configure_association (GstSctpDec * self) "Could not configure SCTP association. Association already in use!"); g_object_unref (self->sctp_association); self->sctp_association = NULL; - goto error; + return FALSE; } self->signal_handler_stream_reset = @@ -521,8 +533,6 @@ configure_association (GstSctpDec * self) on_receive, gst_object_ref (self), gst_object_unref); return TRUE; -error: - return FALSE; } static gboolean @@ -531,17 +541,14 @@ gst_sctp_dec_src_event (GstPad * pad, GstSctpDec * self, GstEvent * event) switch (GST_EVENT_TYPE (event)) { case GST_EVENT_RECONFIGURE: case GST_EVENT_FLUSH_STOP:{ - GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD (pad); - - /* Unflush and start task again */ + GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); + gst_data_queue_set_flushing (sctpdec_pad->packet_queue, FALSE); - gst_pad_start_task (pad, (GstTaskFunction) gst_sctp_data_srcpad_loop, pad, - NULL); return gst_pad_event_default (pad, GST_OBJECT (self), event); } case GST_EVENT_FLUSH_START:{ - GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD (pad); + GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); gst_data_queue_set_flushing (sctpdec_pad->packet_queue, TRUE); gst_data_queue_flush (sctpdec_pad->packet_queue); @@ -553,6 +560,30 @@ gst_sctp_dec_src_event (GstPad * pad, GstSctpDec * self, GstEvent * event) } } +static gboolean +gst_sctp_dec_src_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + GstSctpDec *self = GST_SCTP_DEC_CAST (parent); + GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); + gboolean result; + + GST_DEBUG_OBJECT (self, "activate mode %d active %d", mode, active); + + if (active) { + gst_data_queue_set_flushing (sctpdec_pad->packet_queue, FALSE); + result = + gst_pad_start_task (pad, (GstTaskFunction) gst_sctp_data_srcpad_loop, + pad, NULL); + } else { + gst_data_queue_set_flushing (sctpdec_pad->packet_queue, TRUE); + gst_data_queue_flush (sctpdec_pad->packet_queue); + result = gst_pad_stop_task (pad); + } + + return result; +} + static void send_sticky_events (GstSctpDec * self, GstPad * pad, guint16 stream_id) { @@ -593,8 +624,12 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) return new_pad; } - g_object_get (self->sctp_association, "state", &state, NULL); + if (!self->sctp_association) { + GST_ERROR_OBJECT (self, "Attempt to get pad without a GstSctpAssociation"); + return NULL; + } + g_object_get (self->sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_ERROR_OBJECT (self, "The SCTP association must be established before a new stream can be created"); @@ -614,6 +649,8 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) gst_pad_set_event_function (new_pad, GST_DEBUG_FUNCPTR ((GstPadEventFunction) gst_sctp_dec_src_event)); + gst_pad_set_activatemode_function (new_pad, + GST_DEBUG_FUNCPTR (gst_sctp_dec_src_activate_mode)); if (!gst_pad_set_active (new_pad, TRUE)) goto error_cleanup; @@ -627,9 +664,6 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) gst_flow_combiner_add_pad (self->flow_combiner, new_pad); GST_OBJECT_UNLOCK (self); - gst_pad_start_task (new_pad, (GstTaskFunction) gst_sctp_data_srcpad_loop, - new_pad, NULL); - gst_object_ref (new_pad); return new_pad; @@ -640,20 +674,6 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) return NULL; } -static void -remove_pad (GstSctpDec * self, GstPad * pad) -{ - stop_srcpad_task (pad); - GST_PAD_STREAM_LOCK (pad); - gst_pad_set_active (pad, FALSE); - if (gst_object_has_as_parent (GST_OBJECT (pad), GST_OBJECT (self))) - gst_element_remove_pad (GST_ELEMENT (self), pad); - GST_PAD_STREAM_UNLOCK (pad); - GST_OBJECT_LOCK (self); - gst_flow_combiner_remove_pad (self->flow_combiner, pad); - GST_OBJECT_UNLOCK (self); -} - static void on_gst_sctp_association_stream_reset (GstSctpAssociation * gst_sctp_association, guint16 stream_id, GstSctpDec * self) @@ -684,7 +704,7 @@ static void on_gst_sctp_association_restart (GstSctpAssociation * gst_sctp_association, GstSctpDec * self) { - (void ) gst_sctp_association; + (void) gst_sctp_association; g_signal_emit (self, signals[SIGNAL_ASSOC_RESTART], 0); } @@ -713,7 +733,7 @@ on_receive (GstSctpAssociation * sctp_association, guint8 * buf, "Received incoming packet of size %" G_GSIZE_FORMAT " with stream id %u ppid %u", length, stream_id, ppid); - sctpdec_pad = GST_SCTP_DEC_PAD (src_pad); + sctpdec_pad = GST_SCTP_DEC_PAD_CAST (src_pad); gstbuf = gst_buffer_new_wrapped_full (0, buf, length, 0, length, buf, (GDestroyNotify) usrsctp_freedumpbuffer); @@ -733,46 +753,27 @@ on_receive (GstSctpAssociation * sctp_association, guint8 * buf, } static void -stop_srcpad_task (GstPad * pad) -{ - GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD (pad); - - gst_data_queue_set_flushing (sctpdec_pad->packet_queue, TRUE); - gst_data_queue_flush (sctpdec_pad->packet_queue); - gst_pad_stop_task (pad); -} - -static void -remove_pad_it (const GValue * item, gpointer user_data) -{ - GstPad *pad = g_value_get_object (item); - GstSctpDec *self = user_data; - - remove_pad (self, pad); -} - -static void -stop_all_srcpad_tasks (GstSctpDec * self) -{ - GstIterator *it; - - it = gst_element_iterate_src_pads (GST_ELEMENT (self)); - while (gst_iterator_foreach (it, remove_pad_it, self) == GST_ITERATOR_RESYNC) - gst_iterator_resync (it); - gst_iterator_free (it); -} - -static void -sctpdec_cleanup (GstSctpDec * self) +cleanup_association (GstSctpDec * self) { if (self->sctp_association) { gst_sctp_association_set_on_packet_received (self->sctp_association, NULL, NULL, NULL); - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_association_restart); - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_stream_reset); + + if (self->signal_handler_association_restart != 0) { + g_signal_handler_disconnect (self->sctp_association, + self->signal_handler_association_restart); + self->signal_handler_association_restart = 0; + } + + if (self->signal_handler_stream_reset != 0) { + g_signal_handler_disconnect (self->sctp_association, + self->signal_handler_stream_reset); + self->signal_handler_stream_reset = 0; + } gst_sctp_association_force_close (self->sctp_association); + + g_object_unref (self->sctp_association); + self->sctp_association = NULL; } } diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h index 43269b2629a..80ba5426242 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h @@ -35,6 +35,7 @@ G_BEGIN_DECLS #define GST_TYPE_SCTP_DEC (gst_sctp_dec_get_type()) #define GST_SCTP_DEC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_SCTP_DEC, GstSctpDec)) +#define GST_SCTP_DEC_CAST(obj) (GstSctpDec*)(obj) #define GST_SCTP_DEC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_DEC, GstSctpDecClass)) #define GST_IS_SCTP_DEC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_SCTP_DEC)) #define GST_IS_SCTP_DEC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_SCTP_DEC)) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index c573badfee3..502a4011063 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -83,8 +83,8 @@ static GParamSpec *properties[NUM_PROPERTIES]; GType gst_sctp_enc_pad_get_type (void); #define GST_TYPE_SCTP_ENC_PAD (gst_sctp_enc_pad_get_type()) -#define GST_SCTP_ENC_PAD(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_SCTP_ENC_PAD, GstSctpEncPad)) -#define GST_SCTP_ENC_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_ENC_PAD, GstSctpEncPadClass)) +#define GST_SCTP_ENC_PAD_CAST(obj) (GstSctpEncPad*)(obj) +#define GST_SCTP_ENC_PAD_CAST_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_ENC_PAD, GstSctpEncPadClass)) #define GST_IS_SCTP_ENC_PAD(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_SCTP_ENC_PAD)) #define GST_IS_SCTP_ENC_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_SCTP_ENC_PAD)) @@ -114,7 +114,7 @@ G_DEFINE_TYPE (GstSctpEncPad, gst_sctp_enc_pad, GST_TYPE_PAD); static void gst_sctp_enc_pad_finalize (GObject * object) { - GstSctpEncPad *self = GST_SCTP_ENC_PAD (object); + GstSctpEncPad *self = GST_SCTP_ENC_PAD_CAST (object); g_cond_clear (&self->cond); g_mutex_clear (&self->lock); @@ -157,14 +157,17 @@ static gboolean gst_sctp_enc_sink_event (GstPad * pad, GstObject * parent, GstEvent * event); static gboolean gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event); +static gboolean +gst_sctp_enc_src_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active); static void on_sctp_association_state_changed (GstSctpAssociation * sctp_association, GParamSpec * pspec, GstSctpEnc * self); static gboolean configure_association (GstSctpEnc * self); +static void cleanup_association (GstSctpEnc * self); + static void on_sctp_packet_out (GstSctpAssociation * sctp_association, const guint8 * buf, gsize length, gpointer user_data); -static void stop_srcpad_task (GstPad * pad, GstSctpEnc * self); -static void sctpenc_cleanup (GstSctpEnc * self); static void get_config_from_caps (const GstCaps * caps, gboolean * ordered, GstSctpAssociationPartialReliability * reliability, guint32 * reliability_param, guint32 * ppid, gboolean * ppid_available); @@ -225,8 +228,7 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) properties[PROP_AGGRESSIVE_HEARTBEAT] = g_param_spec_boolean ("aggressive-heartbeat", "Aggressive heartbeat", "When set to TRUE, set the heartbeat interval to 1000ms and the assoc " - "rtx max to 2.", - FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + "rtx max to 2.", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); g_object_class_install_properties (gobject_class, NUM_PROPERTIES, properties); @@ -295,21 +297,37 @@ gst_sctp_enc_init (GstSctpEnc * self) self->src_pad = gst_pad_new_from_static_template (&src_template, "src"); gst_pad_set_event_function (self->src_pad, GST_DEBUG_FUNCPTR ((GstPadEventFunction) gst_sctp_enc_src_event)); + gst_pad_set_activatemode_function (self->src_pad, + GST_DEBUG_FUNCPTR (gst_sctp_enc_src_activate_mode)); gst_element_add_pad (GST_ELEMENT (self), self->src_pad); g_queue_init (&self->pending_pads); self->src_ret = GST_FLOW_FLUSHING; } +static void +remove_sinkpad (const GValue * item, gpointer user_data) +{ + GstSctpEncPad *sctpenc_pad = g_value_get_object (item); + GstSctpEnc *self = user_data; + + gst_sctp_enc_release_pad (GST_ELEMENT (self), GST_PAD (sctpenc_pad)); +} + static void gst_sctp_enc_dispose (GObject * object) { - GstSctpEnc *self = GST_SCTP_ENC (object); + GstSctpEnc *self = GST_SCTP_ENC_CAST (object); - if (self->sctp_association) { - g_object_unref (self->sctp_association); - self->sctp_association = NULL; - } + /* remove all sinkpads */ + GstIterator *it; + it = gst_element_iterate_sink_pads (GST_ELEMENT (self)); + while (gst_iterator_foreach (it, remove_sinkpad, self) == GST_ITERATOR_RESYNC) + gst_iterator_resync (it); + gst_iterator_free (it); + g_queue_clear (&self->pending_pads); + + gst_pad_set_active (self->src_pad, FALSE); G_OBJECT_CLASS (parent_class)->dispose (object); } @@ -317,7 +335,7 @@ gst_sctp_enc_dispose (GObject * object) static void gst_sctp_enc_finalize (GObject * object) { - GstSctpEnc *self = GST_SCTP_ENC (object); + GstSctpEnc *self = GST_SCTP_ENC_CAST (object); g_queue_clear (&self->pending_pads); gst_object_unref (self->outbound_sctp_packet_queue); @@ -329,7 +347,7 @@ static void gst_sctp_enc_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - GstSctpEnc *self = GST_SCTP_ENC (object); + GstSctpEnc *self = GST_SCTP_ENC_CAST (object); switch (prop_id) { case PROP_GST_SCTP_ASSOCIATION_ID: @@ -354,7 +372,7 @@ static void gst_sctp_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - GstSctpEnc *self = GST_SCTP_ENC (object); + GstSctpEnc *self = GST_SCTP_ENC_CAST (object); switch (prop_id) { case PROP_GST_SCTP_ASSOCIATION_ID: @@ -406,61 +424,58 @@ flush_sinkpads (GstSctpEnc * self, gboolean state) static GstStateChangeReturn gst_sctp_enc_change_state (GstElement * element, GstStateChange transition) { - GstSctpEnc *self = GST_SCTP_ENC (element); - GstStateChangeReturn ret = GST_STATE_CHANGE_FAILURE; - gboolean res = TRUE; + GstSctpEnc *self = GST_SCTP_ENC_CAST (element); + GstStateChangeReturn ret; + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); switch (transition) { case GST_STATE_CHANGE_NULL_TO_READY: - break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - self->need_segment = self->need_stream_start_caps = TRUE; - self->src_ret = GST_FLOW_OK; - gst_data_queue_set_flushing (self->outbound_sctp_packet_queue, FALSE); - res = configure_association (self); - break; - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - stop_srcpad_task (self->src_pad, self); - flush_sinkpads (self, TRUE); - self->src_ret = GST_FLOW_FLUSHING; + if (!configure_association (self)) + ret = GST_STATE_CHANGE_FAILURE; break; case GST_STATE_CHANGE_READY_TO_NULL: + cleanup_association (self); break; default: break; } - if (res) - ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + return ret; +} - switch (transition) { - case GST_STATE_CHANGE_NULL_TO_READY: - break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - gst_pad_start_task (self->src_pad, - (GstTaskFunction) gst_sctp_enc_srcpad_loop, self->src_pad, NULL); - break; - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - sctpenc_cleanup (self); - break; - case GST_STATE_CHANGE_READY_TO_NULL: - break; - default: - break; +static gboolean +gst_sctp_enc_src_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + GstSctpEnc *self = GST_SCTP_ENC_CAST (parent); + gboolean result; + + GST_DEBUG_OBJECT (self, "activate mode %d active %d", mode, active); + + if (active) { + self->need_segment = TRUE; + self->need_stream_start_caps = TRUE; + self->src_ret = GST_FLOW_OK; + gst_data_queue_set_flushing (self->outbound_sctp_packet_queue, FALSE); + + result = gst_pad_start_task (pad, + (GstTaskFunction) gst_sctp_enc_srcpad_loop, pad, NULL); + } else { + gst_data_queue_set_flushing (self->outbound_sctp_packet_queue, TRUE); + gst_data_queue_flush (self->outbound_sctp_packet_queue); + result = gst_pad_stop_task (pad); + self->src_ret = GST_FLOW_FLUSHING; } - return ret; + return result; } static GstPad * gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, const gchar * new_pad_name, const GstCaps * caps) { - GstSctpEnc *self = GST_SCTP_ENC (element); + GstSctpEnc *self = GST_SCTP_ENC_CAST (element); GstPad *new_pad = NULL; GstSctpEncPad *sctpenc_pad; guint32 stream_id; @@ -468,8 +483,12 @@ gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, guint32 new_ppid; gboolean is_new_ppid; - g_object_get (self->sctp_association, "state", &state, NULL); + if (!self->sctp_association) { + GST_ERROR_OBJECT (self, "Attempt to get pad without a GstSctpAssociation"); + return NULL; + } + g_object_get (self->sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_ERROR_OBJECT (self, @@ -505,7 +524,7 @@ gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, gst_pad_set_event_function (new_pad, GST_DEBUG_FUNCPTR (gst_sctp_enc_sink_event)); - sctpenc_pad = GST_SCTP_ENC_PAD (new_pad); + sctpenc_pad = GST_SCTP_ENC_PAD_CAST (new_pad); sctpenc_pad->stream_id = stream_id; sctpenc_pad->ppid = DEFAULT_SCTP_PPID; @@ -541,11 +560,11 @@ gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, static void gst_sctp_enc_release_pad (GstElement * element, GstPad * pad) { - GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD (pad); + GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); GstSctpEnc *self; guint stream_id = 0; - self = GST_SCTP_ENC (element); + self = GST_SCTP_ENC_CAST (element); g_mutex_lock (&sctpenc_pad->lock); sctpenc_pad->flushing = TRUE; @@ -567,7 +586,7 @@ gst_sctp_enc_release_pad (GstElement * element, GstPad * pad) static void gst_sctp_enc_srcpad_loop (GstPad * pad) { - GstSctpEnc *self = GST_SCTP_ENC (GST_PAD_PARENT (pad)); + GstSctpEnc *self = GST_SCTP_ENC_CAST (GST_PAD_PARENT (pad)); GstFlowReturn flow_ret; GstDataQueueItem *item; @@ -636,8 +655,8 @@ gst_sctp_enc_srcpad_loop (GstPad * pad) static GstFlowReturn gst_sctp_enc_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) { - GstSctpEnc *self = GST_SCTP_ENC (parent); - GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD (pad); + GstSctpEnc *self = GST_SCTP_ENC_CAST (parent); + GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); GstSctpEncPad *sctpenc_pad_next = NULL; GstMapInfo map; guint32 ppid; @@ -726,10 +745,15 @@ gst_sctp_enc_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) g_mutex_unlock (&sctpenc_pad->lock); - flow_ret = - gst_sctp_association_send_data (self->sctp_association, data, - length, sctpenc_pad->stream_id, ppid, ordered, pr, pr_param, - &bytes_sent); + if (self->sctp_association) { + flow_ret = + gst_sctp_association_send_data (self->sctp_association, data, + length, sctpenc_pad->stream_id, ppid, ordered, pr, pr_param, + &bytes_sent); + } else { + GST_ERROR_OBJECT (self, "No GstSctpAssociation"); + flow_ret = GST_FLOW_ERROR; + } g_mutex_lock (&sctpenc_pad->lock); if (flow_ret != GST_FLOW_OK) { @@ -784,8 +808,8 @@ gst_sctp_enc_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) static gboolean gst_sctp_enc_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { - GstSctpEnc *self = GST_SCTP_ENC (parent); - GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD (pad); + GstSctpEnc *self = GST_SCTP_ENC_CAST (parent); + GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); gboolean ret, is_new_ppid; guint32 new_ppid; @@ -840,7 +864,7 @@ gst_sctp_enc_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) static gboolean gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event) { - GstSctpEnc *self = GST_SCTP_ENC (parent); + GstSctpEnc *self = GST_SCTP_ENC_CAST (parent); gboolean ret; switch (GST_EVENT_TYPE (event)) { @@ -862,8 +886,6 @@ gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event) GST_OBJECT_LOCK (self); self->src_ret = GST_FLOW_OK; GST_OBJECT_UNLOCK (self); - gst_pad_start_task (self->src_pad, - (GstTaskFunction) gst_sctp_enc_srcpad_loop, self->src_pad, NULL); ret = gst_pad_event_default (pad, parent, event); break; @@ -880,6 +902,11 @@ disconnect (GstSctpEnc * self) { gint state; + if (!self->sctp_association) { + GST_ERROR_OBJECT (self, "No GstSctpAssociation"); + return FALSE; + } + g_object_get (self->sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_WARNING_OBJECT (self, "Cannot disconnect in non-CONNECTED state."); @@ -895,6 +922,11 @@ reconnect (GstSctpEnc * self) { gint state; + if (!self->sctp_association) { + GST_ERROR_OBJECT (self, "No GstSctpAssociation"); + return; + } + g_object_get (self->sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { GST_WARNING_OBJECT (self, "Cannot reconnect in non-DISCONNECTED state."); @@ -910,7 +942,6 @@ configure_association (GstSctpEnc * self) gint state; self->sctp_association = gst_sctp_association_get (self->sctp_association_id); - g_object_get (self->sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_NEW) { @@ -918,7 +949,7 @@ configure_association (GstSctpEnc * self) "Could not configure SCTP association. Association already in use!"); g_object_unref (self->sctp_association); self->sctp_association = NULL; - goto error; + return FALSE; } self->signal_handler_state_changed = @@ -938,8 +969,6 @@ configure_association (GstSctpEnc * self) on_sctp_packet_out, gst_object_ref (self), gst_object_unref); return TRUE; -error: - return FALSE; } static void @@ -1029,41 +1058,23 @@ on_sctp_packet_out (GstSctpAssociation * _association, const guint8 * buf, } static void -stop_srcpad_task (GstPad * pad, GstSctpEnc * self) -{ - gst_data_queue_set_flushing (self->outbound_sctp_packet_queue, TRUE); - gst_data_queue_flush (self->outbound_sctp_packet_queue); - gst_pad_stop_task (pad); -} - -static void -remove_sinkpad (const GValue * item, gpointer user_data) +cleanup_association (GstSctpEnc * self) { - GstSctpEncPad *sctpenc_pad = g_value_get_object (item); - GstSctpEnc *self = user_data; - - gst_sctp_enc_release_pad (GST_ELEMENT (self), GST_PAD (sctpenc_pad)); -} - -static void -sctpenc_cleanup (GstSctpEnc * self) -{ - GstIterator *it; - if (self->sctp_association) { gst_sctp_association_set_on_packet_out (self->sctp_association, NULL, NULL, NULL); - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_state_changed); - stop_srcpad_task (self->src_pad, self); + + if (self->signal_handler_state_changed != 0) { + g_signal_handler_disconnect (self->sctp_association, + self->signal_handler_state_changed); + self->signal_handler_state_changed = 0; + } + gst_sctp_association_force_close (self->sctp_association); - } - it = gst_element_iterate_sink_pads (GST_ELEMENT (self)); - while (gst_iterator_foreach (it, remove_sinkpad, self) == GST_ITERATOR_RESYNC) - gst_iterator_resync (it); - gst_iterator_free (it); - g_queue_clear (&self->pending_pads); + g_object_unref (self->sctp_association); + self->sctp_association = NULL; + } } static void @@ -1129,7 +1140,7 @@ on_get_stream_bytes_sent (GstSctpEnc * self, guint stream_id) return 0; } - sctpenc_pad = GST_SCTP_ENC_PAD (pad); + sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); g_mutex_lock (&sctpenc_pad->lock); bytes_sent = sctpenc_pad->bytes_sent; diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index 35a2ce3a0cd..6a0f97b1870 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -34,6 +34,7 @@ G_BEGIN_DECLS #define GST_TYPE_SCTP_ENC (gst_sctp_enc_get_type()) #define GST_SCTP_ENC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_SCTP_ENC, GstSctpEnc)) +#define GST_SCTP_ENC_CAST(obj) (GstSctpEnc*)(obj) #define GST_SCTP_ENC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_ENC, GstSctpEncClass)) #define GST_IS_SCTP_ENC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_SCTP_ENC)) #define GST_IS_SCTP_ENC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_SCTP_ENC)) From 1c500705676f5c1fd67426a53c0876c793f2b2d2 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 28 Sep 2022 13:09:16 +0200 Subject: [PATCH 207/339] SCTP: fixed race condition and memory leak gst_sctp_association_get can call g_object_ref on a destroyed sctp association because the unref calls do not hold the associations_lock. To solve this we add an API to unref the association while holding the lock. Moved the calls to usrsctp_init/finish to get/unref() respectively to not depend on GObjects behavior, keep the counting of the associations in the init/finalize. Also added g_hash_table_destroy() calls to the association hash table after the last association is destroyed. --- .../gst-plugins-bad/ext/sctp/gstsctpdec.c | 4 +- .../gst-plugins-bad/ext/sctp/gstsctpenc.c | 4 +- .../ext/sctp/sctpassociation.c | 108 ++++++++++-------- .../ext/sctp/sctpassociation.h | 1 + 4 files changed, 68 insertions(+), 49 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index 857061a06d6..a2cfa9a28f5 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -513,7 +513,7 @@ configure_association (GstSctpDec * self) if (state != GST_SCTP_ASSOCIATION_STATE_NEW) { GST_WARNING_OBJECT (self, "Could not configure SCTP association. Association already in use!"); - g_object_unref (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; return FALSE; } @@ -772,7 +772,7 @@ cleanup_association (GstSctpDec * self) } gst_sctp_association_force_close (self->sctp_association); - g_object_unref (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; } } diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 502a4011063..649d69b465c 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -947,7 +947,7 @@ configure_association (GstSctpEnc * self) if (state != GST_SCTP_ASSOCIATION_STATE_NEW) { GST_WARNING_OBJECT (self, "Could not configure SCTP association. Association already in use!"); - g_object_unref (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; return FALSE; } @@ -1072,7 +1072,7 @@ cleanup_association (GstSctpEnc * self) gst_sctp_association_force_close (self->sctp_association); - g_object_unref (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; } } diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 2e4e17010b0..071d02bd7c3 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -210,51 +210,16 @@ gst_usrsctp_debug (const gchar * format, ...) static void gst_sctp_association_init (GstSctpAssociation * self) { - /* No need to lock mutex here as long as the function is only called from gst_sctp_association_get */ - if (number_of_associations == 0) { -#if defined(SCTP_DEBUG) && !defined(GST_DISABLE_GST_DEBUG) - usrsctp_init (0, sctp_packet_out, gst_usrsctp_debug); -#else - usrsctp_init (0, sctp_packet_out, NULL); -#endif - - /* Explicit Congestion Notification */ - usrsctp_sysctl_set_sctp_ecn_enable (0); - - /* Do not send ABORTs in response to INITs (1). - * Do not send ABORTs for received Out of the Blue packets (2). - */ - usrsctp_sysctl_set_sctp_blackhole (2); - - /* Enable interleaving messages for different streams (incoming) - * See: https://tools.ietf.org/html/rfc6458#section-8.1.20 - */ - usrsctp_sysctl_set_sctp_default_frag_interleave (2); - - usrsctp_sysctl_set_sctp_nr_outgoing_streams_default - (DEFAULT_NUMBER_OF_SCTP_STREAMS); - -#if defined(SCTP_DEBUG) && !defined(GST_DISABLE_GST_DEBUG) - if (USRSCTP_GST_DEBUG_LEVEL <= GST_LEVEL_MAX - && USRSCTP_GST_DEBUG_LEVEL <= _gst_debug_min - && USRSCTP_GST_DEBUG_LEVEL <= - gst_debug_category_get_threshold (gst_sctp_debug_category)) { - usrsctp_sysctl_set_sctp_debug_on (SCTP_DEBUG_ALL); - } -#endif - } number_of_associations++; self->local_port = DEFAULT_LOCAL_SCTP_PORT; self->remote_port = DEFAULT_REMOTE_SCTP_PORT; self->sctp_ass_sock = NULL; - - g_mutex_init (&self->association_mutex); - self->state = GST_SCTP_ASSOCIATION_STATE_NEW; - self->use_sock_stream = TRUE; + g_mutex_init (&self->association_mutex); + usrsctp_register_address ((void *) self); } @@ -263,19 +228,15 @@ gst_sctp_association_finalize (GObject * object) { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (object); - G_LOCK (associations_lock); + usrsctp_deregister_address ((void *) self); + /* no need to hold the association_lock, it is held under + gst_sctp_association_unref */ g_hash_table_remove (associations_by_id, GUINT_TO_POINTER (self->association_id)); g_hash_table_remove (ids_by_association, self); - usrsctp_deregister_address ((void *) self); number_of_associations--; - if (number_of_associations == 0) { - usrsctp_finish (); - } - - G_UNLOCK (associations_lock); g_mutex_clear (&self->association_mutex); @@ -375,6 +336,41 @@ gst_sctp_association_get_property (GObject * object, guint prop_id, } } +static void +gst_sctp_association_usrsctp_init (void) +{ +#if defined(SCTP_DEBUG) && !defined(GST_DISABLE_GST_DEBUG) + usrsctp_init (0, sctp_packet_out, gst_usrsctp_debug); +#else + usrsctp_init (0, sctp_packet_out, NULL); +#endif + + /* Explicit Congestion Notification */ + usrsctp_sysctl_set_sctp_ecn_enable (0); + + /* Do not send ABORTs in response to INITs (1). + * Do not send ABORTs for received Out of the Blue packets (2). + */ + usrsctp_sysctl_set_sctp_blackhole (2); + + /* Enable interleaving messages for different streams (incoming) + * See: https://tools.ietf.org/html/rfc6458#section-8.1.20 + */ + usrsctp_sysctl_set_sctp_default_frag_interleave (2); + + usrsctp_sysctl_set_sctp_nr_outgoing_streams_default + (DEFAULT_NUMBER_OF_SCTP_STREAMS); + +#if defined(SCTP_DEBUG) && !defined(GST_DISABLE_GST_DEBUG) + if (USRSCTP_GST_DEBUG_LEVEL <= GST_LEVEL_MAX + && USRSCTP_GST_DEBUG_LEVEL <= _gst_debug_min + && USRSCTP_GST_DEBUG_LEVEL <= + gst_debug_category_get_threshold (gst_sctp_debug_category)) { + usrsctp_sysctl_set_sctp_debug_on (SCTP_DEBUG_ALL); + } +#endif +} + /* Public functions */ GstSctpAssociation * @@ -388,7 +384,12 @@ gst_sctp_association_get (guint32 association_id) GST_DEBUG_CATEGORY_INIT (gst_sctp_debug_category, "sctplib", 0, "debug category for messages from usrsctp"); + if (number_of_associations == 0) + gst_sctp_association_usrsctp_init (); + if (!associations_by_id) { + g_assert (ids_by_association == NULL); + associations_by_id = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, NULL); ids_by_association = @@ -413,7 +414,24 @@ gst_sctp_association_get (guint32 association_id) return association; } -gboolean +void +gst_sctp_association_unref (GstSctpAssociation * self) +{ + G_LOCK (associations_lock); + g_object_unref (self); + + if (number_of_associations == 0) { + usrsctp_finish (); + + g_hash_table_destroy (associations_by_id); + g_hash_table_destroy (ids_by_association); + associations_by_id = NULL; + ids_by_association = NULL; + } + G_UNLOCK (associations_lock); +} + +static gboolean gst_sctp_association_start_unlocked (GstSctpAssociation * self) { if (self->state != GST_SCTP_ASSOCIATION_STATE_READY && diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 9afa601804f..c74c3890e28 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -106,6 +106,7 @@ struct _GstSctpAssociationClass GType gst_sctp_association_get_type (void); GstSctpAssociation *gst_sctp_association_get (guint32 association_id); +void gst_sctp_association_unref (GstSctpAssociation * self); gboolean gst_sctp_association_start (GstSctpAssociation * self); void gst_sctp_association_set_on_packet_out (GstSctpAssociation * self, From 2754bf7ebdb3ff76b900a65a6208127895425921 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 29 Sep 2022 17:13:19 +0200 Subject: [PATCH 208/339] SCTP: add the srcpad to the sctpdec element before activing it This fixes a potential NULL access on the pads parent inside gst_sctp_data_srcpad_loop (cherry picked from commit fc409b9d4d73ee481327b0cba1343ffa426b7ea2) --- subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index a2cfa9a28f5..60d4318bdc6 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -652,13 +652,14 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) gst_pad_set_activatemode_function (new_pad, GST_DEBUG_FUNCPTR (gst_sctp_dec_src_activate_mode)); + if (!gst_element_add_pad (GST_ELEMENT (self), new_pad)) + goto error_add; + if (!gst_pad_set_active (new_pad, TRUE)) goto error_cleanup; send_sticky_events (self, new_pad, stream_id); - if (!gst_element_add_pad (GST_ELEMENT (self), new_pad)) - goto error_add; GST_OBJECT_LOCK (self); gst_flow_combiner_add_pad (self->flow_combiner, new_pad); From 15560e40fbcffd0d548faeda6746190553d37f84 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 19 Dec 2022 10:43:39 +0100 Subject: [PATCH 209/339] SCTP: build with invariants checks in To help debug memory bugs --- subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build index 6a1ab845da9..e1f57ee7bad 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build @@ -143,7 +143,7 @@ if have_sconn_len endif # Options -if false +if true compile_args += ['-DINVARIANTS'] endif if not gst_debug_disabled From 1772f977c4b4dddb95faba5bfd79a64d8ab2cdfb Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 28 Nov 2022 12:07:54 +0100 Subject: [PATCH 210/339] SCTP: added null checks --- .../ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c | 4 ++++ .../gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c | 2 ++ 2 files changed, 6 insertions(+) diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c index a494b2c0ae4..ccd7a55f444 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c @@ -7385,6 +7385,10 @@ sctp_sendall(struct sctp_inpcb *inp, struct uio *uio, struct mbuf *m, int ret; struct sctp_copy_all *ca; + if (!uio) { + return (EFAULT); + } + if (inp->sctp_flags & SCTP_PCB_FLAGS_SND_ITERATOR_UP) { /* There is another. */ return (EBUSY); diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c index b72b05ebc30..ce9d922cb1b 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c @@ -1880,6 +1880,8 @@ soconnect(struct socket *so, struct sockaddr *nam) */ if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && (error = sodisconnect(so))) { error = EISCONN; + } else if (nam == NULL){ + return EINVAL; } else { /* * Prevent accumulated error from previous connection from From 9e9e3ef5d9f145bf25dc321fb44570614129472c Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 28 Nov 2022 12:34:10 +0100 Subject: [PATCH 211/339] SCTP: cleaned up code in gst_sctp_association_change_state Changed code to have gst_sctp_association_change_state() and unlocked() version. --- .../ext/sctp/sctpassociation.c | 56 ++++++++----------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 071d02bd7c3..a8de81e7049 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -133,10 +133,10 @@ static void handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid); static void maybe_set_state_to_ready (GstSctpAssociation * self); -static void gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, +static void gst_sctp_association_change_state_unlocked (GstSctpAssociation * + self, GstSctpAssociationState new_state); +static void gst_sctp_association_change_state (GstSctpAssociation * self, GstSctpAssociationState new_state); -static gboolean gst_sctp_association_change_state (GstSctpAssociation * self, - GstSctpAssociationState new_state, gboolean lock); static void gst_sctp_association_class_init (GstSctpAssociationClass * klass) @@ -300,7 +300,8 @@ maybe_set_state_to_ready (GstSctpAssociation * self) if ((self->state == GST_SCTP_ASSOCIATION_STATE_NEW) && (self->local_port != 0 && self->remote_port != 0) && (self->packet_out_cb != NULL) && (self->packet_received_cb != NULL)) { - gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_READY); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_READY); } g_mutex_unlock (&self->association_mutex); } @@ -981,7 +982,7 @@ handle_notification (GstSctpAssociation * self, case SCTP_SHUTDOWN_EVENT: GST_DEBUG_OBJECT (self, "Event: SCTP_SHUTDOWN_EVENT"); gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTING, TRUE); + GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); break; case SCTP_ADAPTATION_INDICATION: GST_DEBUG_OBJECT (self, "Event: SCTP_ADAPTATION_INDICATION"); @@ -1100,7 +1101,8 @@ handle_association_changed (GstSctpAssociation * self, break; case SCTP_CANT_STR_ASSOC: GST_WARNING_OBJECT (self, "SCTP event SCTP_CANT_STR_ASSOC received"); - gst_sctp_association_change_state (self, GST_SCTP_ASSOCIATION_STATE_ERROR, TRUE); + gst_sctp_association_change_state (self, + GST_SCTP_ASSOCIATION_STATE_ERROR); break; } } @@ -1145,39 +1147,29 @@ handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, static void gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, GstSctpAssociationState new_state) - { - /* Association mutex is held on entry */ - gboolean notify = gst_sctp_association_change_state (self, new_state, FALSE); +{ + gboolean notify = FALSE; + + if (self->state != new_state + && self->state != GST_SCTP_ASSOCIATION_STATE_ERROR) { + self->state = new_state; + notify = TRUE; + } + + /* Unlock the mutex to emit the property change event to avoid deadlock + * if the client calls back into this object from its event handler. */ if (notify) { - /* Unlock the mutex to emit the property change event to avoid deadlock - * if the client calls back into this object from its event handler. */ g_mutex_unlock (&self->association_mutex); g_object_notify_by_pspec (G_OBJECT (self), properties[PROP_STATE]); g_mutex_lock (&self->association_mutex); } } -/* Returns TRUE if lock==FALSE and notification is needed later. - * Takes the mutex shortly if lock==TRUE! */ -static gboolean +static void gst_sctp_association_change_state (GstSctpAssociation * self, - GstSctpAssociationState new_state, gboolean lock) + GstSctpAssociationState new_state) { - if (lock) - g_mutex_lock (&self->association_mutex); - if (self->state != new_state - && self->state != GST_SCTP_ASSOCIATION_STATE_ERROR) { - self->state = new_state; - if (lock) { - g_mutex_unlock (&self->association_mutex); - g_object_notify_by_pspec (G_OBJECT (self), properties[PROP_STATE]); - return FALSE; - } else { - return TRUE; - } - } else { - if (lock) - g_mutex_unlock (&self->association_mutex); - return FALSE; - } + g_mutex_lock (&self->association_mutex); + gst_sctp_association_change_state_unlocked (self, new_state); + g_mutex_unlock (&self->association_mutex); } From 61dfeb91d9ede05646368253bcfd59c2b3d8853e Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 28 Nov 2022 12:46:27 +0100 Subject: [PATCH 212/339] SCTP: applied gst-indent --- .../ext/sctp/sctpassociation.c | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index a8de81e7049..0f96ea95afe 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -187,8 +187,7 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) properties[PROP_AGGRESSIVE_HEARTBEAT] = g_param_spec_boolean ("aggressive-heartbeat", "Aggressive heartbeat", "When set to TRUE, set the heartbeat interval to 10ms and the assoc " - "rtx max to 1.", - FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + "rtx max to 1.", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); g_object_class_install_properties (gobject_class, NUM_PROPERTIES, properties); } @@ -399,7 +398,7 @@ gst_sctp_association_get (guint32 association_id) association = g_hash_table_lookup (associations_by_id, - GUINT_TO_POINTER (association_id)); + GUINT_TO_POINTER (association_id)); if (!association) { association = g_object_new (GST_SCTP_TYPE_ASSOCIATION, "association-id", @@ -609,7 +608,7 @@ gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id) srs->srs_assoc_id = self->sctp_assoc_id; if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, - SCTP_RESET_STREAMS, srs, length) < 0) { + SCTP_RESET_STREAMS, srs, length) < 0) { GST_INFO_OBJECT (self, "Resetting stream id=%u failed", stream_id); } } @@ -666,7 +665,6 @@ gst_sctp_association_disconnect_unlocked (GstSctpAssociation * self, } /* Fall through to ensure the transition to disconnected occurs */ - if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { force_close_unlocked (self, FALSE); gst_sctp_association_change_state_unlocked (self, @@ -1032,7 +1030,7 @@ _apply_aggressive_heartbeat_unlocked (GstSctpAssociation * self) assoc_params.sasoc_assoc_id = self->sctp_assoc_id; assoc_params.sasoc_asocmaxrxt = 1; if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, - SCTP_ASSOCINFO, &assoc_params, sizeof (assoc_params))) { + SCTP_ASSOCINFO, &assoc_params, sizeof (assoc_params))) { GST_WARNING_OBJECT (self, "Could not set SCTP_ASSOCINFO"); } @@ -1042,34 +1040,36 @@ _apply_aggressive_heartbeat_unlocked (GstSctpAssociation * self) peer_addr_params.spp_flags = SPP_HB_ENABLE; peer_addr_params.spp_hbinterval = 10; if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, - SCTP_PEER_ADDR_PARAMS, &peer_addr_params, sizeof (peer_addr_params))) { + SCTP_PEER_ADDR_PARAMS, &peer_addr_params, + sizeof (peer_addr_params))) { GST_WARNING_OBJECT (self, "Could not set SCTP_PEER_ADDR_PARAMS"); } } static void handle_sctp_comm_up (GstSctpAssociation * self, - const struct sctp_assoc_change * sac) + const struct sctp_assoc_change *sac) { GST_INFO_OBJECT (self, "SCTP_COMM_UP"); g_mutex_lock (&self->association_mutex); - if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { - self->sctp_assoc_id = sac->sac_assoc_id; - _apply_aggressive_heartbeat_unlocked (self); - gst_sctp_association_change_state_unlocked (self, - GST_SCTP_ASSOCIATION_STATE_CONNECTED); - GST_INFO_OBJECT (self, "SCTP association connected!"); - } else if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - GST_INFO_OBJECT (self, "SCTP association already open"); - } else { - GST_WARNING_OBJECT (self, "SCTP association in unexpected state: %d", self->state); - } + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { + self->sctp_assoc_id = sac->sac_assoc_id; + _apply_aggressive_heartbeat_unlocked (self); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_CONNECTED); + GST_INFO_OBJECT (self, "SCTP association connected!"); + } else if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + GST_INFO_OBJECT (self, "SCTP association already open"); + } else { + GST_WARNING_OBJECT (self, "SCTP association in unexpected state: %d", + self->state); + } g_mutex_unlock (&self->association_mutex); } static void handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, - const struct sctp_assoc_change * sac) + const struct sctp_assoc_change *sac) { GST_INFO_OBJECT (self, "SCTP event %s received", sac->sac_state == SCTP_COMM_LOST ? From 36109e69fd3d2edddced60e0e908faac5414231a Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 28 Nov 2022 13:16:58 +0100 Subject: [PATCH 213/339] SCTP: ensure thread-safe access in gst_sctp_association_reset_stream We need to ensure that members like state, sctp_assoc_id and sctp_ass_sock are accessed with a lock. --- .../ext/sctp/sctpassociation.c | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 0f96ea95afe..b12a8ce7455 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -599,20 +599,28 @@ gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id) struct sctp_reset_streams *srs; socklen_t length; + g_mutex_lock (&self->association_mutex); + + if (self->state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + /* only allow resets on connected streams */ + g_mutex_unlock (&self->association_mutex); + return; + } + length = (socklen_t) (sizeof (struct sctp_reset_streams) + sizeof (guint16)); srs = (struct sctp_reset_streams *) g_malloc0 (length); srs->srs_flags = SCTP_STREAM_RESET_OUTGOING; srs->srs_number_streams = 1; srs->srs_stream_list[0] = stream_id; - srs->srs_assoc_id = self->sctp_assoc_id; - if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, - SCTP_RESET_STREAMS, srs, length) < 0) { - GST_INFO_OBJECT (self, "Resetting stream id=%u failed", stream_id); - } + + if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, + SCTP_RESET_STREAMS, srs, length) < 0) { + GST_WARNING_OBJECT (self, "Resetting stream id=%u failed", stream_id); } + g_mutex_unlock (&self->association_mutex); + g_free (srs); } From 3a17d695e51709b99caef5487971692dc1b52095 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 28 Nov 2022 15:03:03 +0100 Subject: [PATCH 214/339] SCTP: access sctp_ass_sock with a lock in gst_sctp_association_send_data To be consistent with the rest of the code. --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index b12a8ce7455..406f2b8486a 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -542,7 +542,6 @@ gst_sctp_association_send_data (GstSctpAssociation * self, const guint8 * buf, } } remote_addr = get_sctp_socket_address (self, self->remote_port); - g_mutex_unlock (&self->association_mutex); /* TODO: We probably want to split too large chunks into multiple packets * and only set the SCTP_EOR flag on the last one. Firefox is using 0x4000 @@ -571,6 +570,9 @@ gst_sctp_association_send_data (GstSctpAssociation * self, const guint8 * buf, usrsctp_sendv (self->sctp_ass_sock, buf, length, (struct sockaddr *) &remote_addr, 1, (void *) &spa, (socklen_t) sizeof (struct sctp_sendv_spa), SCTP_SENDV_SPA, 0); + + g_mutex_unlock (&self->association_mutex); + if (bytes_sent < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK) { bytes_sent = 0; From 346c98bbd6445748cdbd5fdda5715cf35bd39ab8 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 28 Nov 2022 16:10:15 +0100 Subject: [PATCH 215/339] SCTP: removed manual counting of associations, use the hash table directly instead --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 406f2b8486a..d444734d001 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -102,7 +102,6 @@ static GParamSpec *properties[NUM_PROPERTIES]; G_LOCK_DEFINE_STATIC (associations_lock); static GHashTable *associations_by_id = NULL; static GHashTable *ids_by_association = NULL; -static guint32 number_of_associations = 0; /* Interface implementations */ static void gst_sctp_association_finalize (GObject * object); @@ -209,8 +208,6 @@ gst_usrsctp_debug (const gchar * format, ...) static void gst_sctp_association_init (GstSctpAssociation * self) { - number_of_associations++; - self->local_port = DEFAULT_LOCAL_SCTP_PORT; self->remote_port = DEFAULT_REMOTE_SCTP_PORT; self->sctp_ass_sock = NULL; @@ -235,8 +232,6 @@ gst_sctp_association_finalize (GObject * object) GUINT_TO_POINTER (self->association_id)); g_hash_table_remove (ids_by_association, self); - number_of_associations--; - g_mutex_clear (&self->association_mutex); G_OBJECT_CLASS (gst_sctp_association_parent_class)->finalize (object); @@ -384,9 +379,6 @@ gst_sctp_association_get (guint32 association_id) GST_DEBUG_CATEGORY_INIT (gst_sctp_debug_category, "sctplib", 0, "debug category for messages from usrsctp"); - if (number_of_associations == 0) - gst_sctp_association_usrsctp_init (); - if (!associations_by_id) { g_assert (ids_by_association == NULL); @@ -394,6 +386,8 @@ gst_sctp_association_get (guint32 association_id) g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, NULL); ids_by_association = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, NULL); + + gst_sctp_association_usrsctp_init (); } association = @@ -420,13 +414,13 @@ gst_sctp_association_unref (GstSctpAssociation * self) G_LOCK (associations_lock); g_object_unref (self); - if (number_of_associations == 0) { - usrsctp_finish (); - + if (g_hash_table_size (associations_by_id) == 0) { g_hash_table_destroy (associations_by_id); g_hash_table_destroy (ids_by_association); associations_by_id = NULL; ids_by_association = NULL; + + usrsctp_finish (); } G_UNLOCK (associations_lock); } From 986dddeb9db8f3780aed856892fcc756e19219a4 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 20 Dec 2022 13:24:31 +0100 Subject: [PATCH 216/339] SCTP: replace g_object_bind_property with g_object_set This fixes a very nasty double-free on the SCTP association in which a SCTP association weak-reference will be dangling around and the elements unref() will land in an unlocked gst_sctp_association_finalize() call. Other changes: - Added locking around the SCTP association in both encoder and decoder. - Added ref() API to the SCTP association, to be consistent with unref(). - Calling SCTP associations API with a ref() of the current association to safely avoid finalizing a last reference. - Moved maybe_set_state_to_ready to an unlocked() version, since in the contexts it is called there is no need to release and acquire the lock. --- .../gst-plugins-bad/ext/sctp/gstsctpdec.c | 94 +++++++++---- .../gst-plugins-bad/ext/sctp/gstsctpdec.h | 2 + .../gst-plugins-bad/ext/sctp/gstsctpenc.c | 123 +++++++++++++----- .../gst-plugins-bad/ext/sctp/gstsctpenc.h | 2 + .../ext/sctp/sctpassociation.c | 30 +++-- .../ext/sctp/sctpassociation.h | 2 + 6 files changed, 184 insertions(+), 69 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index 60d4318bdc6..3c29568ad17 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -77,6 +77,10 @@ static GParamSpec *properties[NUM_PROPERTIES]; #define MAX_GST_SCTP_ASSOCIATION_ID 65535 #define MAX_STREAM_ID 65535 +#define GST_SCTP_DEC_GET_ASSOC_MUTEX(self) (&self->association_mutex) +#define GST_SCTP_DEC_ASSOC_MUTEX_LOCK(self) (g_mutex_lock (GST_SCTP_DEC_GET_ASSOC_MUTEX (self))) +#define GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK(self) (g_mutex_unlock (GST_SCTP_DEC_GET_ASSOC_MUTEX (self))) + GType gst_sctp_dec_pad_get_type (void); #define GST_TYPE_SCTP_DEC_PAD (gst_sctp_dec_pad_get_type()) @@ -233,9 +237,11 @@ gst_sctp_dec_class_init (GstSctpDecClass * klass) static void gst_sctp_dec_init (GstSctpDec * self) { + g_mutex_init (GST_SCTP_DEC_GET_ASSOC_MUTEX (self)); + + self->sctp_association = NULL; self->sctp_association_id = DEFAULT_GST_SCTP_ASSOCIATION_ID; self->local_sctp_port = DEFAULT_LOCAL_SCTP_PORT; - self->flow_combiner = gst_flow_combiner_new (); self->sink_pad = gst_pad_new_from_static_template (&sink_template, "sink"); @@ -294,6 +300,8 @@ gst_sctp_dec_finalize (GObject * object) gst_flow_combiner_free (self->flow_combiner); self->flow_combiner = NULL; + g_mutex_clear (GST_SCTP_DEC_GET_ASSOC_MUTEX (self)); + G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -309,6 +317,13 @@ gst_sctp_dec_set_property (GObject * object, guint prop_id, break; case PROP_LOCAL_SCTP_PORT: self->local_sctp_port = g_value_get_uint (value); + + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + if (self->sctp_association) { + g_object_set (self->sctp_association, "local-port", + self->local_sctp_port, NULL); + } + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); @@ -365,6 +380,7 @@ gst_sctp_dec_packet_chain (GstPad * pad, GstSctpDec * self, GstBuffer * buf) { GstFlowReturn flow_ret; GstMapInfo map; + GstSctpAssociation *sctp_association = NULL; GST_DEBUG_OBJECT (self, "Processing received buffer %" GST_PTR_FORMAT, buf); @@ -374,10 +390,16 @@ gst_sctp_dec_packet_chain (GstPad * pad, GstSctpDec * self, GstBuffer * buf) return GST_FLOW_ERROR; } - if (self->sctp_association != NULL) { - gst_sctp_association_incoming_packet (self->sctp_association, + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + + if (sctp_association) { + gst_sctp_association_incoming_packet (sctp_association, (const guint8 *) map.data, (guint32) map.size); + gst_sctp_association_unref (sctp_association); } + gst_buffer_unmap (buf, &map); gst_buffer_unref (buf); @@ -507,6 +529,7 @@ configure_association (GstSctpDec * self) { gint state; + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); self->sctp_association = gst_sctp_association_get (self->sctp_association_id); g_object_get (self->sctp_association, "state", &state, NULL); @@ -515,6 +538,7 @@ configure_association (GstSctpDec * self) "Could not configure SCTP association. Association already in use!"); gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); return FALSE; } @@ -526,12 +550,14 @@ configure_association (GstSctpDec * self) g_signal_connect_object (self->sctp_association, "association-restart", G_CALLBACK (on_gst_sctp_association_restart), self, 0); - g_object_bind_property (self, "local-sctp-port", self->sctp_association, - "local-port", G_BINDING_SYNC_CREATE); + g_object_set (self->sctp_association, "local-port", self->local_sctp_port, + NULL); gst_sctp_association_set_on_packet_received (self->sctp_association, on_receive, gst_object_ref (self), gst_object_unref); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + return TRUE; } @@ -542,7 +568,7 @@ gst_sctp_dec_src_event (GstPad * pad, GstSctpDec * self, GstEvent * event) case GST_EVENT_RECONFIGURE: case GST_EVENT_FLUSH_STOP:{ GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); - + gst_data_queue_set_flushing (sctpdec_pad->packet_queue, FALSE); return gst_pad_event_default (pad, GST_OBJECT (self), event); @@ -624,12 +650,16 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) return new_pad; } + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); if (!self->sctp_association) { GST_ERROR_OBJECT (self, "Attempt to get pad without a GstSctpAssociation"); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); return NULL; } g_object_get (self->sctp_association, "state", &state, NULL); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_ERROR_OBJECT (self, "The SCTP association must be established before a new stream can be created"); @@ -756,34 +786,44 @@ on_receive (GstSctpAssociation * sctp_association, guint8 * buf, static void cleanup_association (GstSctpDec * self) { - if (self->sctp_association) { - gst_sctp_association_set_on_packet_received (self->sctp_association, NULL, - NULL, NULL); - - if (self->signal_handler_association_restart != 0) { - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_association_restart); - self->signal_handler_association_restart = 0; - } + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + if (!self->sctp_association) { + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + return; + } - if (self->signal_handler_stream_reset != 0) { - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_stream_reset); - self->signal_handler_stream_reset = 0; - } - gst_sctp_association_force_close (self->sctp_association); + if (self->signal_handler_association_restart != 0) { + g_signal_handler_disconnect (self->sctp_association, + self->signal_handler_association_restart); + self->signal_handler_association_restart = 0; + } - gst_sctp_association_unref (self->sctp_association); - self->sctp_association = NULL; + if (self->signal_handler_stream_reset != 0) { + g_signal_handler_disconnect (self->sctp_association, + self->signal_handler_stream_reset); + self->signal_handler_stream_reset = 0; } + + gst_sctp_association_set_on_packet_received (self->sctp_association, NULL, + NULL, NULL); + gst_sctp_association_force_close (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); + self->sctp_association = NULL; + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); } static void on_reset_stream (GstSctpDec * self, guint stream_id) { - if (self->sctp_association) { - gst_sctp_association_reset_stream (self->sctp_association, stream_id); - on_gst_sctp_association_stream_reset (self->sctp_association, stream_id, - self); + GstSctpAssociation *sctp_association = NULL; + + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + + if (sctp_association) { + gst_sctp_association_reset_stream (sctp_association, stream_id); + on_gst_sctp_association_stream_reset (sctp_association, stream_id, self); + gst_sctp_association_unref (sctp_association); } } diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h index 80ba5426242..82eb5d5db5a 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h @@ -46,6 +46,8 @@ struct _GstSctpDec { GstElement element; + GMutex association_mutex; + GstFlowCombiner *flow_combiner; GstPad *sink_pad; diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 649d69b465c..57faeeb5567 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -80,6 +80,10 @@ static GParamSpec *properties[NUM_PROPERTIES]; #define BUFFER_FULL_SLEEP_TIME 100000 +#define GST_SCTP_ENC_GET_ASSOC_MUTEX(self) (&self->association_mutex) +#define GST_SCTP_ENC_ASSOC_MUTEX_LOCK(self) (g_mutex_lock (GST_SCTP_ENC_GET_ASSOC_MUTEX (self))) +#define GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK(self) (g_mutex_unlock (GST_SCTP_ENC_GET_ASSOC_MUTEX (self))) + GType gst_sctp_enc_pad_get_type (void); #define GST_TYPE_SCTP_ENC_PAD (gst_sctp_enc_pad_get_type()) @@ -286,10 +290,12 @@ data_queue_full_cb (GstDataQueue * queue, gpointer user_data) static void gst_sctp_enc_init (GstSctpEnc * self) { + g_mutex_init (GST_SCTP_ENC_GET_ASSOC_MUTEX (self)); + + self->sctp_association = NULL; self->sctp_association_id = DEFAULT_GST_SCTP_ASSOCIATION_ID; self->remote_sctp_port = DEFAULT_REMOTE_SCTP_PORT; - self->sctp_association = NULL; self->outbound_sctp_packet_queue = gst_data_queue_new (data_queue_check_full_cb, data_queue_full_cb, data_queue_empty_cb, NULL); @@ -340,6 +346,8 @@ gst_sctp_enc_finalize (GObject * object) g_queue_clear (&self->pending_pads); gst_object_unref (self->outbound_sctp_packet_queue); + g_mutex_clear (GST_SCTP_ENC_GET_ASSOC_MUTEX (self)); + G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -355,12 +363,30 @@ gst_sctp_enc_set_property (GObject * object, guint prop_id, break; case PROP_REMOTE_SCTP_PORT: self->remote_sctp_port = g_value_get_uint (value); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (self->sctp_association) { + g_object_set (self->sctp_association, "remote-port", + self->remote_sctp_port, NULL); + } + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); break; case PROP_USE_SOCK_STREAM: self->use_sock_stream = g_value_get_boolean (value); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (self->sctp_association) { + g_object_set (self->sctp_association, "use-sock-stream", + self->use_sock_stream, NULL); + } + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); break; case PROP_AGGRESSIVE_HEARTBEAT: self->aggressive_heartbeat = g_value_get_boolean (value); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (self->sctp_association) { + g_object_set (self->sctp_association, "aggressive-heartbeat", + self->aggressive_heartbeat, NULL); + } + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); @@ -483,12 +509,16 @@ gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, guint32 new_ppid; gboolean is_new_ppid; + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); if (!self->sctp_association) { GST_ERROR_OBJECT (self, "Attempt to get pad without a GstSctpAssociation"); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); return NULL; } g_object_get (self->sctp_association, "state", &state, NULL); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_ERROR_OBJECT (self, @@ -563,6 +593,7 @@ gst_sctp_enc_release_pad (GstElement * element, GstPad * pad) GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); GstSctpEnc *self; guint stream_id = 0; + GstSctpAssociation *sctp_association = NULL; self = GST_SCTP_ENC_CAST (element); @@ -574,8 +605,14 @@ gst_sctp_enc_release_pad (GstElement * element, GstPad * pad) stream_id = sctpenc_pad->stream_id; gst_pad_set_active (pad, FALSE); - if (self->sctp_association) - gst_sctp_association_reset_stream (self->sctp_association, stream_id); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + + if (sctp_association) { + gst_sctp_association_reset_stream (sctp_association, stream_id); + gst_sctp_association_unref (sctp_association); + } GST_PAD_STREAM_LOCK (pad); if (gst_object_has_as_parent (GST_OBJECT (pad), GST_OBJECT (element))) @@ -742,14 +779,20 @@ gst_sctp_enc_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) while (!sctpenc_pad->flushing) { guint32 bytes_sent; + GstSctpAssociation *sctp_association = NULL; g_mutex_unlock (&sctpenc_pad->lock); - if (self->sctp_association) { + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + + if (sctp_association) { flow_ret = - gst_sctp_association_send_data (self->sctp_association, data, + gst_sctp_association_send_data (sctp_association, data, length, sctpenc_pad->stream_id, ppid, ordered, pr, pr_param, &bytes_sent); + gst_sctp_association_unref (sctp_association); } else { GST_ERROR_OBJECT (self, "No GstSctpAssociation"); flow_ret = GST_FLOW_ERROR; @@ -901,19 +944,27 @@ static gboolean disconnect (GstSctpEnc * self) { gint state; + GstSctpAssociation *sctp_association = NULL; - if (!self->sctp_association) { + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + + if (!sctp_association) { GST_ERROR_OBJECT (self, "No GstSctpAssociation"); return FALSE; } - g_object_get (self->sctp_association, "state", &state, NULL); + g_object_get (sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_WARNING_OBJECT (self, "Cannot disconnect in non-CONNECTED state."); + gst_sctp_association_unref (sctp_association); return FALSE; } - gst_sctp_association_disconnect (self->sctp_association); + gst_sctp_association_disconnect (sctp_association); + gst_sctp_association_unref (sctp_association); + return TRUE; } @@ -921,19 +972,26 @@ static void reconnect (GstSctpEnc * self) { gint state; + GstSctpAssociation *sctp_association = NULL; - if (!self->sctp_association) { + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + + if (!sctp_association) { GST_ERROR_OBJECT (self, "No GstSctpAssociation"); return; } - g_object_get (self->sctp_association, "state", &state, NULL); + g_object_get (sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { GST_WARNING_OBJECT (self, "Cannot reconnect in non-DISCONNECTED state."); + gst_sctp_association_unref (sctp_association); return; } - gst_sctp_association_start (self->sctp_association); + gst_sctp_association_start (sctp_association); + gst_sctp_association_unref (sctp_association); } static gboolean @@ -941,6 +999,7 @@ configure_association (GstSctpEnc * self) { gint state; + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); self->sctp_association = gst_sctp_association_get (self->sctp_association_id); g_object_get (self->sctp_association, "state", &state, NULL); @@ -949,6 +1008,7 @@ configure_association (GstSctpEnc * self) "Could not configure SCTP association. Association already in use!"); gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); return FALSE; } @@ -956,17 +1016,13 @@ configure_association (GstSctpEnc * self) g_signal_connect_object (self->sctp_association, "notify::state", G_CALLBACK (on_sctp_association_state_changed), self, 0); - g_object_bind_property (self, "remote-sctp-port", self->sctp_association, - "remote-port", G_BINDING_SYNC_CREATE); - - g_object_bind_property (self, "use-sock-stream", self->sctp_association, - "use-sock-stream", G_BINDING_SYNC_CREATE); - - g_object_bind_property (self, "aggressive-heartbeat", self->sctp_association, - "aggressive-heartbeat", G_BINDING_SYNC_CREATE); + g_object_set (self->sctp_association, "remote-port", self->remote_sctp_port, + "use-sock-stream", self->use_sock_stream, "aggressive-heartbeat", + self->aggressive_heartbeat, NULL); gst_sctp_association_set_on_packet_out (self->sctp_association, on_sctp_packet_out, gst_object_ref (self), gst_object_unref); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); return TRUE; } @@ -1060,21 +1116,24 @@ on_sctp_packet_out (GstSctpAssociation * _association, const guint8 * buf, static void cleanup_association (GstSctpEnc * self) { - if (self->sctp_association) { - gst_sctp_association_set_on_packet_out (self->sctp_association, NULL, NULL, - NULL); - - if (self->signal_handler_state_changed != 0) { - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_state_changed); - self->signal_handler_state_changed = 0; - } - - gst_sctp_association_force_close (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (!self->sctp_association) { + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + return; + } - gst_sctp_association_unref (self->sctp_association); - self->sctp_association = NULL; + if (self->signal_handler_state_changed != 0) { + g_signal_handler_disconnect (self->sctp_association, + self->signal_handler_state_changed); + self->signal_handler_state_changed = 0; } + + gst_sctp_association_set_on_packet_out (self->sctp_association, NULL, NULL, + NULL); + gst_sctp_association_force_close (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); + self->sctp_association = NULL; + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); } static void diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index 6a0f97b1870..acc1fee31a3 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -46,6 +46,8 @@ struct _GstSctpEnc { GstElement element; + GMutex association_mutex; + GstPad *src_pad; GstFlowReturn src_ret; gboolean need_stream_start_caps, need_segment; diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index d444734d001..095c6475437 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -131,7 +131,7 @@ static void handle_stream_reset_event (GstSctpAssociation * self, static void handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid); -static void maybe_set_state_to_ready (GstSctpAssociation * self); +static void maybe_set_state_to_ready_unlocked (GstSctpAssociation * self); static void gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, GstSctpAssociationState new_state); static void gst_sctp_association_change_state (GstSctpAssociation * self, @@ -277,9 +277,10 @@ gst_sctp_association_set_property (GObject * object, guint prop_id, break; } - g_mutex_unlock (&self->association_mutex); if (prop_id == PROP_LOCAL_PORT || prop_id == PROP_REMOTE_PORT) - maybe_set_state_to_ready (self); + maybe_set_state_to_ready_unlocked (self); + + g_mutex_unlock (&self->association_mutex); return; @@ -288,16 +289,14 @@ gst_sctp_association_set_property (GObject * object, guint prop_id, } static void -maybe_set_state_to_ready (GstSctpAssociation * self) +maybe_set_state_to_ready_unlocked (GstSctpAssociation * self) { - g_mutex_lock (&self->association_mutex); if ((self->state == GST_SCTP_ASSOCIATION_STATE_NEW) && (self->local_port != 0 && self->remote_port != 0) && (self->packet_out_cb != NULL) && (self->packet_received_cb != NULL)) { gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_READY); } - g_mutex_unlock (&self->association_mutex); } static void @@ -408,6 +407,17 @@ gst_sctp_association_get (guint32 association_id) return association; } +GstSctpAssociation * +gst_sctp_association_ref (GstSctpAssociation * self) +{ + GstSctpAssociation *ref = NULL; + G_LOCK (associations_lock); + if (self) + ref = g_object_ref (self); + G_UNLOCK (associations_lock); + return ref; +} + void gst_sctp_association_unref (GstSctpAssociation * self) { @@ -479,9 +489,8 @@ gst_sctp_association_set_on_packet_out (GstSctpAssociation * self, self->packet_out_cb = packet_out_cb; self->packet_out_user_data = user_data; self->packet_out_destroy_notify = destroy_notify; + maybe_set_state_to_ready_unlocked (self); g_mutex_unlock (&self->association_mutex); - - maybe_set_state_to_ready (self); } void @@ -497,9 +506,8 @@ gst_sctp_association_set_on_packet_received (GstSctpAssociation * self, self->packet_received_cb = packet_received_cb; self->packet_received_user_data = user_data; self->packet_received_destroy_notify = destroy_notify; + maybe_set_state_to_ready_unlocked (self); g_mutex_unlock (&self->association_mutex); - - maybe_set_state_to_ready (self); } void @@ -1163,9 +1171,11 @@ gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, /* Unlock the mutex to emit the property change event to avoid deadlock * if the client calls back into this object from its event handler. */ if (notify) { + gst_sctp_association_ref (self); g_mutex_unlock (&self->association_mutex); g_object_notify_by_pspec (G_OBJECT (self), properties[PROP_STATE]); g_mutex_lock (&self->association_mutex); + gst_sctp_association_unref (self); } } diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index c74c3890e28..434c8a85b7b 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -106,6 +106,8 @@ struct _GstSctpAssociationClass GType gst_sctp_association_get_type (void); GstSctpAssociation *gst_sctp_association_get (guint32 association_id); + +GstSctpAssociation *gst_sctp_association_ref (GstSctpAssociation * self); void gst_sctp_association_unref (GstSctpAssociation * self); gboolean gst_sctp_association_start (GstSctpAssociation * self); From 1fa6536c07f168714de083c045207176ff0d734b Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 1 Oct 2021 00:41:36 +0200 Subject: [PATCH 217/339] gst-plugins-base/rtpdummyhdrextimpl: unused variable --- .../gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c b/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c index 8eee22d2047..fda8efde6cd 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c @@ -219,7 +219,8 @@ gst_rtp_dummy_hdr_ext_set_caps_from_attributes (GstRTPHeaderExtension * ext, static gboolean gst_rtp_dummy_hdr_ext_set_attributes (GstRTPHeaderExtension * ext, - GstRTPHeaderExtensionDirection direction, const gchar * attributes) + G_GNUC_UNUSED GstRTPHeaderExtensionDirection direction, + const gchar * attributes) { GstRTPDummyHdrExt *dummy = GST_RTP_DUMMY_HDR_EXT (ext); From a8bf667d39bade3ffa19b729fd489311994c01f8 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 6 Oct 2021 16:14:49 +0200 Subject: [PATCH 218/339] bytewriter: fix msvc warnings --- subprojects/gstreamer/libs/gst/base/gstbytewriter.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/base/gstbytewriter.h b/subprojects/gstreamer/libs/gst/base/gstbytewriter.h index 365c7742d73..f2ba7530ba5 100644 --- a/subprojects/gstreamer/libs/gst/base/gstbytewriter.h +++ b/subprojects/gstreamer/libs/gst/base/gstbytewriter.h @@ -374,7 +374,7 @@ gst_byte_writer_put_buffer_unchecked (GstByteWriter * writer, GstBuffer * buffer gst_buffer_extract (buffer, offset, (guint8 *) & writer->parent.data[writer->parent.byte], size); - writer->parent.byte += size; + writer->parent.byte += (guint) size; writer->parent.size = MAX (writer->parent.size, writer->parent.byte); } @@ -394,7 +394,7 @@ _gst_byte_writer_put_buffer_inline (GstByteWriter * writer, GstBuffer * buffer, size -= offset; } - if (G_UNLIKELY (!_gst_byte_writer_ensure_free_space_inline (writer, size))) + if (G_UNLIKELY (!_gst_byte_writer_ensure_free_space_inline (writer, (guint) size))) return FALSE; gst_byte_writer_put_buffer_unchecked (writer, buffer, offset, size); From 0d026dc38fbc060f17fd2389ba00462f511c41a2 Mon Sep 17 00:00:00 2001 From: Will Miller Date: Fri, 8 Oct 2021 17:55:53 +0100 Subject: [PATCH 219/339] [pexhack] re-add gst_debug_syslog option Changes lost in monorepo merge; originally from 7bbcd29fad3f1f6e0c6a34345fd19f7092ffa0ae --- subprojects/gstreamer/gst/gstinfo.c | 7 ++----- subprojects/gstreamer/meson.build | 5 +++++ subprojects/gstreamer/meson_options.txt | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/subprojects/gstreamer/gst/gstinfo.c b/subprojects/gstreamer/gst/gstinfo.c index 75789a37559..7515e065ad9 100644 --- a/subprojects/gstreamer/gst/gstinfo.c +++ b/subprojects/gstreamer/gst/gstinfo.c @@ -1687,14 +1687,11 @@ gst_debug_log_default (GstDebugCategory * category, GstDebugLevel level, #define PRINT_FMT " "PID_FMT" "PTR_FMT" %s "CAT_FMT" %s\n" syslog (LOG_INFO|LOG_LOCAL3, "%" GST_TIME_FORMAT PRINT_FMT, GST_TIME_ARGS (elapsed), pid, g_thread_self (), gst_debug_level_get_name (level), - gst_debug_category_get_name (category), file, line, function, obj, - gst_debug_message_get (message)); + gst_debug_category_get_name (category), file, line, function, + object_id ? object_id : "", gst_debug_message_get (message)); #undef PRINT_FMT } #endif - - if (object != NULL) - g_free (obj); } /** diff --git a/subprojects/gstreamer/meson.build b/subprojects/gstreamer/meson.build index f0d3f742f8b..a48488fba36 100644 --- a/subprojects/gstreamer/meson.build +++ b/subprojects/gstreamer/meson.build @@ -491,6 +491,11 @@ if not gst_debug add_project_arguments(['-Wno-unused'], language: 'c') endif +gst_debug_syslog = get_option('gst_debug_syslog') +if gst_debug_syslog + add_project_arguments(['-DENABLE_GST_DEBUG_SYSLOG'], language: 'c') +endif + warning_flags = [ '-Wmissing-declarations', '-Wmissing-prototypes', diff --git a/subprojects/gstreamer/meson_options.txt b/subprojects/gstreamer/meson_options.txt index 3a34bcaaa31..f201726e83f 100644 --- a/subprojects/gstreamer/meson_options.txt +++ b/subprojects/gstreamer/meson_options.txt @@ -1,4 +1,5 @@ option('gst_debug', type : 'boolean', value : true) +option('gst_debug_syslog', type : 'boolean', value : false) option('gst_parse', type : 'boolean', value : true, description: 'Enable pipeline string parser') option('registry', type : 'boolean', value : true) From d02693308f032bcbb407226039405e1cb47ce2d1 Mon Sep 17 00:00:00 2001 From: Thomas Williams Date: Wed, 15 Sep 2021 15:25:11 +0100 Subject: [PATCH 220/339] gstharness: make removal of peer pad thread safe --- subprojects/gstreamer/libs/gst/check/gstharness.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/check/gstharness.c b/subprojects/gstreamer/libs/gst/check/gstharness.c index 03a135f49e8..1aad6f5f140 100644 --- a/subprojects/gstreamer/libs/gst/check/gstharness.c +++ b/subprojects/gstreamer/libs/gst/check/gstharness.c @@ -1122,8 +1122,11 @@ gst_harness_teardown (GstHarness * h) h->sink_harness = NULL; if (h->srcpad) { - if (gst_pad_is_request_pad (GST_PAD_PEER (h->srcpad))) - gst_element_release_request_pad (h->element, GST_PAD_PEER (h->srcpad)); + GstPad *peer_srcpad = gst_pad_get_peer (h->srcpad); + if (gst_pad_is_request_pad (peer_srcpad)) + gst_element_release_request_pad (h->element, peer_srcpad); + if (peer_srcpad) + gst_object_unref (peer_srcpad); g_free (priv->element_sinkpad_name); gst_pad_set_active (h->srcpad, FALSE); @@ -1139,8 +1142,11 @@ gst_harness_teardown (GstHarness * h) h->srcpad = NULL; if (h->sinkpad) { - if (gst_pad_is_request_pad (GST_PAD_PEER (h->sinkpad))) - gst_element_release_request_pad (h->element, GST_PAD_PEER (h->sinkpad)); + GstPad *peer_sinkpad = gst_pad_get_peer (h->sinkpad); + if (gst_pad_is_request_pad (peer_sinkpad)) + gst_element_release_request_pad (h->element, peer_sinkpad); + if (peer_sinkpad) + gst_object_unref (peer_sinkpad); g_free (priv->element_srcpad_name); gst_pad_set_active (h->sinkpad, FALSE); From 4b1022cec4f720072bb6ea46386456d2bf590fff Mon Sep 17 00:00:00 2001 From: Frederik Vestre Date: Mon, 24 Jan 2022 13:03:30 +0100 Subject: [PATCH 221/339] gstharness: Limited support for bufferlists. --- .../gstreamer/libs/gst/check/gstharness.c | 312 +++++++++++++++++- .../gstreamer/libs/gst/check/gstharness.h | 9 + .../gstreamer/tests/check/libs/gstharness.c | 102 +++++- 3 files changed, 410 insertions(+), 13 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/check/gstharness.c b/subprojects/gstreamer/libs/gst/check/gstharness.c index 1aad6f5f140..7d2d5f08f3b 100644 --- a/subprojects/gstreamer/libs/gst/check/gstharness.c +++ b/subprojects/gstreamer/libs/gst/check/gstharness.c @@ -1,6 +1,6 @@ /* GstHarness - A test-harness for GStreamer testing * - * Copyright (C) 2012-2015 Pexip + * Copyright (C) 2012-2021 Pexip * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -132,7 +132,12 @@ #include #include +GST_DEBUG_CATEGORY (gst_harness_debug); +#define GST_CAT_DEFAULT gst_harness_debug + static void gst_harness_stress_free (GstHarnessThread * t); +static GstBuffer *gst_harness_extract_one_buffer_locked (GstHarness * h, + GstMiniObject * queue_element); /* Keys used for storing and retrieving associations to pads and elements with * g_object_set_data (): */ @@ -229,6 +234,10 @@ gst_harness_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) g_mutex_lock (&priv->blocking_push_mutex); g_atomic_int_inc (&priv->recv_buffers); + GST_TRACE_OBJECT (pad, + "receiving chain buffer: harness: %" GST_PTR_FORMAT " buf: %" + GST_PTR_FORMAT, h, buffer); + if (priv->drop_buffers) { gst_buffer_unref (buffer); } else { @@ -248,6 +257,47 @@ gst_harness_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) return GST_FLOW_OK; } +static GstFlowReturn +gst_harness_chain_list (GstPad * pad, GstObject * parent, + GstBufferList * buffer_list) +{ + GstHarness *h; + GstHarnessLink *link; + GstHarnessPrivate *priv; + guint listlen; + (void) parent; + + if (!(link = gst_harness_pad_link_lock (pad, &h))) + return GST_FLOW_FLUSHING; + g_assert (h != NULL); + priv = h->priv; + + g_mutex_lock (&priv->blocking_push_mutex); + listlen = gst_buffer_list_length (buffer_list); + g_atomic_int_add (&priv->recv_buffers, listlen); + + GST_TRACE_OBJECT (pad, + "receiving chain list harness: %" GST_PTR_FORMAT " #buffers: (%" + G_GUINT32_FORMAT ") %" GST_PTR_FORMAT, h, listlen, buffer_list); + + if (priv->drop_buffers) { + gst_buffer_list_unref (buffer_list); + } else { + g_mutex_lock (&priv->buf_or_eos_mutex); + g_async_queue_push (priv->buffer_queue, buffer_list); + g_cond_signal (&priv->buf_or_eos_cond); + g_mutex_unlock (&priv->buf_or_eos_mutex); + } + + if (priv->blocking_push_mode) { + g_cond_wait (&priv->blocking_push_cond, &priv->blocking_push_mutex); + } + g_mutex_unlock (&priv->blocking_push_mutex); + + gst_harness_link_unlock (link); + return GST_FLOW_OK; +} + static gboolean gst_harness_src_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -643,6 +693,7 @@ gst_harness_setup_sink_pad (GstHarness * h, gst_harness_pad_link_set (h->sinkpad, h); gst_pad_set_chain_function (h->sinkpad, gst_harness_chain); + gst_pad_set_chain_list_function (h->sinkpad, gst_harness_chain_list); gst_pad_set_query_function (h->sinkpad, gst_harness_sink_query); gst_pad_set_event_function (h->sinkpad, gst_harness_sink_event); @@ -719,6 +770,8 @@ gst_harness_new_empty (void) GstHarness *h; GstHarnessPrivate *priv; + GST_DEBUG_CATEGORY_INIT (gst_harness_debug, "gstharness", 0, "GstHarness"); + h = g_new0 (GstHarness, 1); g_assert (h != NULL); h->priv = g_new0 (GstHarnessPrivate, 1); @@ -733,7 +786,7 @@ gst_harness_new_empty (void) priv->testclock = GST_TEST_CLOCK_CAST (gst_test_clock_new ()); priv->buffer_queue = g_async_queue_new_full ( - (GDestroyNotify) gst_buffer_unref); + (GDestroyNotify) gst_mini_object_unref); priv->src_event_queue = g_async_queue_new_full ( (GDestroyNotify) gst_event_unref); priv->sink_event_queue = g_async_queue_new_full ( @@ -1087,6 +1140,7 @@ gst_harness_new_parse (const gchar * launchline) return h; } + /** * gst_harness_teardown: * @h: a #GstHarness @@ -1155,6 +1209,7 @@ gst_harness_teardown (GstHarness * h) * they try to access this harness through pad data */ gst_harness_pad_link_tear_down (h->sinkpad); gst_pad_set_chain_function (h->sinkpad, NULL); + gst_pad_set_chain_list_function (h->sinkpad, NULL); gst_pad_set_event_function (h->sinkpad, NULL); gst_pad_set_query_function (h->sinkpad, NULL); @@ -1710,6 +1765,33 @@ gst_harness_push (GstHarness * h, GstBuffer * buffer) return gst_pad_push (h->srcpad, buffer); } +/** + * gst_harness_push_list: + * @h: a #GstHarness + * @buffer_list: (transfer full): a #GstBufferList to push + * + * Pushes a #GstBufferList on the #GstHarness srcpad. The standard way of + * interacting with an harnessed element. + * + * MT safe. + * + * Returns: a #GstFlowReturn with the result from the push + * + */ +GstFlowReturn +gst_harness_push_list (GstHarness * h, GstBufferList * buffer_list) +{ + guint32 listlen; + GstHarnessPrivate *priv = h->priv; + g_assert (buffer_list != NULL); + listlen = gst_buffer_list_length (buffer_list); + if (listlen > 0) { + GstBuffer *last_buffer = gst_buffer_list_get (buffer_list, listlen - 1); + priv->last_push_ts = GST_BUFFER_TIMESTAMP (last_buffer); + } + return gst_pad_push_list (h->srcpad, buffer_list); +} + /** * gst_harness_pull: * @h: a #GstHarness @@ -1727,9 +1809,17 @@ gst_harness_push (GstHarness * h, GstBuffer * buffer) GstBuffer * gst_harness_pull (GstHarness * h) { - GstHarnessPrivate *priv = h->priv; - GstBuffer *buf = (GstBuffer *) g_async_queue_timeout_pop (priv->buffer_queue, - G_USEC_PER_SEC * 60); + GstHarnessPrivate *priv; + GstMiniObject *queue_element; + GstBuffer *buf; + + priv = h->priv; + queue_element = + GST_MINI_OBJECT_CAST (g_async_queue_timeout_pop (priv->buffer_queue, + G_USEC_PER_SEC * 60)); + g_mutex_lock (&priv->buf_or_eos_mutex); + buf = gst_harness_extract_one_buffer_locked (h, queue_element); + g_mutex_unlock (&priv->buf_or_eos_mutex); if (priv->blocking_push_mode) { g_mutex_lock (&priv->blocking_push_mutex); @@ -1757,13 +1847,15 @@ gst_harness_pull (GstHarness * h) gboolean gst_harness_pull_until_eos (GstHarness * h, GstBuffer ** buf) { + GstMiniObject *queue_element; GstHarnessPrivate *priv = h->priv; gboolean success = TRUE; gint64 end_time = g_get_monotonic_time () + 60 * G_TIME_SPAN_SECOND; g_mutex_lock (&priv->buf_or_eos_mutex); while (success) { - *buf = g_async_queue_try_pop (priv->buffer_queue); + queue_element = g_async_queue_try_pop (priv->buffer_queue); + *buf = gst_harness_extract_one_buffer_locked (h, queue_element); if (*buf || priv->eos_received) break; success = g_cond_wait_until (&priv->buf_or_eos_cond, @@ -1774,6 +1866,56 @@ gst_harness_pull_until_eos (GstHarness * h, GstBuffer ** buf) return success; } +static void +gst_harness_async_queue_transfer (GAsyncQueue * new, GAsyncQueue * old) +{ + gpointer popped; + while ((popped = g_async_queue_try_pop (old)) != NULL) { + g_async_queue_push (new, popped); + } +} + +static GstBuffer * +gst_harness_extract_one_buffer_locked (GstHarness * h, + GstMiniObject * queue_element) +{ + GstHarnessPrivate *priv; + GAsyncQueue *old_buffer_queue; + GstBuffer *buf = NULL; + priv = h->priv; + if (queue_element != NULL) { + if (GST_IS_BUFFER (queue_element)) { + buf = GST_BUFFER_CAST (queue_element); + } else { + guint32 listlen; + GstBufferList *list = GST_BUFFER_LIST_CAST (queue_element); + g_assert (GST_BUFFER_LIST (queue_element)); + + listlen = gst_buffer_list_length (list); + if (listlen > 0) { + buf = gst_buffer_ref (gst_buffer_list_get (list, 0)); + } + if (listlen > 1) { + /* List has multiple elements, remove the first one and transfer the + rest of the list as separate buffers to a new queue */ + gst_buffer_list_remove (list, 0, 1); + listlen--; + old_buffer_queue = priv->buffer_queue; + priv->buffer_queue = g_async_queue_new_full ( + (GDestroyNotify) gst_mini_object_unref); + for (int i = 0; i < listlen; ++i) { + g_async_queue_push (priv->buffer_queue, + gst_buffer_ref (gst_buffer_list_get (list, i))); + } + gst_harness_async_queue_transfer (priv->buffer_queue, old_buffer_queue); + g_async_queue_unref (old_buffer_queue); + } + gst_buffer_list_unref (list); + } + } + return buf; +} + /** * gst_harness_try_pull: * @h: a #GstHarness @@ -1791,8 +1933,20 @@ gst_harness_pull_until_eos (GstHarness * h, GstBuffer ** buf) GstBuffer * gst_harness_try_pull (GstHarness * h) { - GstHarnessPrivate *priv = h->priv; - GstBuffer *buf = (GstBuffer *) g_async_queue_try_pop (priv->buffer_queue); + GstHarnessPrivate *priv; + GstBuffer *buf; + GstMiniObject *queue_element; + + g_return_val_if_fail (h != NULL, NULL); + + priv = h->priv; + + queue_element = + GST_MINI_OBJECT_CAST (g_async_queue_try_pop (priv->buffer_queue)); + + g_mutex_lock (&priv->buf_or_eos_mutex); + buf = gst_harness_extract_one_buffer_locked (h, queue_element); + g_mutex_unlock (&priv->buf_or_eos_mutex); if (priv->blocking_push_mode) { g_mutex_lock (&priv->blocking_push_mutex); @@ -1861,8 +2015,27 @@ gst_harness_buffers_received (GstHarness * h) guint gst_harness_buffers_in_queue (GstHarness * h) { + gpointer popped; + GAsyncQueue *old_buffer_queue; GstHarnessPrivate *priv = h->priv; - return g_async_queue_length (priv->buffer_queue); + guint enqueued_buffers = 0; + + g_mutex_lock (&priv->buf_or_eos_mutex); + old_buffer_queue = priv->buffer_queue; + priv->buffer_queue = g_async_queue_new_full ( + (GDestroyNotify) gst_mini_object_unref); + while ((popped = g_async_queue_try_pop (old_buffer_queue)) != NULL) { + if (GST_IS_BUFFER_LIST (popped)) { + GstBufferList *popped_list = GST_BUFFER_LIST_CAST (popped); + enqueued_buffers += gst_buffer_list_length (popped_list); + } else { + enqueued_buffers++; + } + g_async_queue_push (priv->buffer_queue, popped); + } + g_async_queue_unref (old_buffer_queue); + g_mutex_unlock (&priv->buf_or_eos_mutex); + return enqueued_buffers; } /** @@ -1900,6 +2073,7 @@ gst_harness_take_all_data_as_buffer (GstHarness * h) { GstHarnessPrivate *priv; GstBuffer *ret, *buf; + GstMiniObject *queue_element; g_return_val_if_fail (h != NULL, NULL); @@ -1907,16 +2081,23 @@ gst_harness_take_all_data_as_buffer (GstHarness * h) g_async_queue_lock (priv->buffer_queue); - ret = g_async_queue_try_pop_unlocked (priv->buffer_queue); + queue_element = g_async_queue_try_pop_unlocked (priv->buffer_queue); - if (ret == NULL) { + if (queue_element == NULL) { ret = gst_buffer_new (); } else { + /* Extracting one element and pushing the rest to a queue inside the same + function is not very efficient, but we are not efficient in appending + buffers either, so let's stick with the KISS theme. */ + ret = gst_harness_extract_one_buffer_locked (h, queue_element); /* buffer appending isn't very efficient for larger numbers of buffers * or lots of memories, but this function is not performance critical and * we can still improve it if and when the need arises. For now KISS. */ - while ((buf = g_async_queue_try_pop_unlocked (priv->buffer_queue))) + while ((queue_element = + g_async_queue_try_pop_unlocked (priv->buffer_queue))) { + buf = gst_harness_extract_one_buffer_locked (h, queue_element); ret = gst_buffer_append (ret, buf); + } } g_async_queue_unlock (priv->buffer_queue); @@ -1977,6 +2158,113 @@ gst_harness_take_all_data_as_bytes (GstHarness * h) } +/** + * gst_harness_pull_list + * @h: a #GstHarness + * + * Pulls a #GstBufferList from the #GAsyncQueue on the #GstHarness sinkpad. + * The pull will timeout in 60 seconds. This is the standard way of getting + * a buffer list from a harnessed #GstElement. + * + * If a normal buffer is submitted to the GstHarness this function will return + * null as long as that buffer is at the head of the queue + * (use #gst_harness_pull to remove it) + * + * MT safe. + * + * Returns: (transfer full): a #GstBufferList or %NULL if timed out, + * or buffer lists are not enabled. + */ +GstBufferList * +gst_harness_pull_list (GstHarness * h) +{ + GAsyncQueue *old_buffer_queue; + GstHarnessPrivate *priv; + GstMiniObject *queue_element; + g_return_val_if_fail (h != NULL, NULL); + priv = h->priv; + + queue_element = + GST_MINI_OBJECT_CAST (g_async_queue_timeout_pop (priv->buffer_queue, + G_USEC_PER_SEC * 60)); + g_mutex_lock (&priv->buf_or_eos_mutex); + if (queue_element != NULL && GST_IS_BUFFER (queue_element)) { + /* Head of queue is a buffer, add it back to the queue */ + old_buffer_queue = priv->buffer_queue; + priv->buffer_queue = g_async_queue_new_full ( + (GDestroyNotify) gst_mini_object_unref); + g_async_queue_push (priv->buffer_queue, queue_element); + gst_harness_async_queue_transfer (priv->buffer_queue, old_buffer_queue); + queue_element = NULL; + } + g_mutex_unlock (&priv->buf_or_eos_mutex); + + g_assert (queue_element == NULL || GST_IS_BUFFER_LIST (queue_element)); + + if (priv->blocking_push_mode) { + g_mutex_lock (&priv->blocking_push_mutex); + g_cond_signal (&priv->blocking_push_cond); + g_mutex_unlock (&priv->blocking_push_mutex); + } + + return GST_BUFFER_LIST_CAST (queue_element); +} + +/** + * gst_harness_try_pull_list: + * @h: a #GstHarness + * + * Pulls a #GstBuffer from the #GAsyncQueue on the #GstHarness sinkpad. Unlike + * gst_harness_pull this will not wait for any buffers if not any are present, + * and return %NULL straight away. + * + * Pulls a #GstBufferList from the #GAsyncQueue on the #GstHarness sinkpad. + * Unlike gst_harness_pull_list this will not wait for any buffers if not any + * are present, and return %NULL straight away. + * This function is only relevant if #gst_harness_buffer_list_support_set_enable + * is set to true, otherwise NULL will be returned. + * + * MT safe. + * + * Returns: (transfer full): a #GstBuffer or %NULL if no buffers are present in + * the #GAsyncQueue, or buffer lists not enabled. + * + */ +GstBufferList * +gst_harness_try_pull_list (GstHarness * h) +{ + GAsyncQueue *old_buffer_queue; + GstMiniObject *queue_element; + GstHarnessPrivate *priv; + g_return_val_if_fail (h != NULL, NULL); + priv = h->priv; + + g_mutex_lock (&priv->buf_or_eos_mutex); + + queue_element = g_async_queue_try_pop (priv->buffer_queue); + + if (queue_element != NULL && GST_IS_BUFFER (queue_element)) { + /* Head of queue is a buffer, add it back to the queue */ + old_buffer_queue = priv->buffer_queue; + priv->buffer_queue = g_async_queue_new_full ( + (GDestroyNotify) gst_mini_object_unref); + g_async_queue_push (priv->buffer_queue, queue_element); + gst_harness_async_queue_transfer (priv->buffer_queue, old_buffer_queue); + g_async_queue_unref (old_buffer_queue); + queue_element = NULL; + } + + g_mutex_unlock (&priv->buf_or_eos_mutex); + + if (priv->blocking_push_mode) { + g_mutex_lock (&priv->blocking_push_mutex); + g_cond_signal (&priv->blocking_push_cond); + g_mutex_unlock (&priv->blocking_push_mutex); + } + + return GST_BUFFER_LIST_CAST (queue_element); +} + /** * gst_harness_dump_to_file: * @h: a #GstHarness diff --git a/subprojects/gstreamer/libs/gst/check/gstharness.h b/subprojects/gstreamer/libs/gst/check/gstharness.h index 160fdb01d8e..4b310b6f42b 100644 --- a/subprojects/gstreamer/libs/gst/check/gstharness.h +++ b/subprojects/gstreamer/libs/gst/check/gstharness.h @@ -182,6 +182,9 @@ GstBuffer * gst_harness_create_buffer (GstHarness * h, gsize size); GST_CHECK_API GstFlowReturn gst_harness_push (GstHarness * h, GstBuffer * buffer); +GST_CHECK_API +GstFlowReturn gst_harness_push_list (GstHarness * h, GstBufferList * buffer_list); + GST_CHECK_API GstBuffer * gst_harness_pull (GstHarness * h); @@ -218,6 +221,12 @@ GBytes * gst_harness_take_all_data_as_bytes (GstHarness * h); GST_CHECK_API GstClockTime gst_harness_get_last_pushed_timestamp (GstHarness * h); +GST_CHECK_API +GstBufferList * gst_harness_pull_list (GstHarness * h); + +GST_CHECK_API +GstBufferList *gst_harness_try_pull_list (GstHarness * h); + /* downstream events */ GST_CHECK_API diff --git a/subprojects/gstreamer/tests/check/libs/gstharness.c b/subprojects/gstreamer/tests/check/libs/gstharness.c index 7999b4bb944..ac26f5cf577 100644 --- a/subprojects/gstreamer/tests/check/libs/gstharness.c +++ b/subprojects/gstreamer/tests/check/libs/gstharness.c @@ -1,7 +1,7 @@ /* * Tests and examples of GstHarness * - * Copyright (C) 2015 Havard Graff + * Copyright (C) 2015-2021 Havard Graff * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -22,6 +22,8 @@ #include "config.h" #endif +#include + #include #include @@ -293,6 +295,103 @@ GST_START_TEST (test_get_all_data) GST_END_TEST; +static GstFlowReturn +gst_harness_dummy_identity_chain_list (GstPad * pad, GstObject * parent, + GstBufferList * buffer_list) +{ + GstBaseTransform *trans = GST_BASE_TRANSFORM_CAST (parent); + return gst_pad_push_list (trans->srcpad, buffer_list); +} + +GST_START_TEST (test_src_harness_buflist) +{ + GstBaseTransform *trans; + GstBufferList *list, *push_list; + GstHarness *h; + GstBuffer *buffer; + + h = gst_harness_new ("identity"); + gst_harness_set_src_caps_str (h, "mycaps"); + + /* Make GstIdentity element forward buffer lists as is instead of destructing + them into individual buffers */ + trans = GST_BASE_TRANSFORM_CAST (gst_harness_find_element (h, "identity")); + gst_pad_set_chain_list_function (trans->sinkpad, + GST_DEBUG_FUNCPTR (gst_harness_dummy_identity_chain_list)); + + /* Push and pull list */ + push_list = gst_buffer_list_new_sized (2); + for (int i = 0; i < 2; ++i) { + buffer = gst_buffer_new_allocate (NULL, i*17, NULL); + gst_buffer_list_add (push_list, buffer); + } + + gst_harness_push_list (h, push_list); + + /* Verify that identity outputs a buffer by pulling and unreffing */ + list = gst_harness_pull_list (h); + fail_unless (list != NULL); + fail_unless_equals_int (gst_buffer_list_length (list), 2); + for (int i = 0; i < 2; ++i) { + buffer = gst_buffer_list_get(list, i); + fail_unless_equals_int(gst_buffer_get_size(buffer), i*17) + } + gst_buffer_list_unref (list); + + /* Push list, pull individual buffers */ + push_list = gst_buffer_list_new_sized (3); + for (int i = 0; i < 3; ++i) { + buffer = gst_buffer_new_allocate (NULL, i * 127, NULL); + gst_buffer_list_add (push_list, buffer); + } + + gst_harness_push_list (h, push_list); + + for (int i = 0; i < 3; ++i) { + buffer = gst_harness_pull (h); + fail_unless_equals_int (gst_buffer_get_size (buffer), i * 127); + fail_unless (buffer != NULL); + gst_buffer_unref (buffer); + } + + /* Push a buffer */ + buffer = gst_buffer_new_allocate (NULL, 384, NULL); + gst_harness_push (h, buffer); + + /* Push a list */ + push_list = gst_buffer_list_new_sized (4); + for (int i = 0; i < 4; ++i) { + buffer = gst_buffer_new_allocate (NULL, i * 937, NULL); + gst_buffer_list_add (push_list, buffer); + } + gst_harness_push_list (h, push_list); + + /* Try to pull list, which should fail as a buffer is at the front of the queue. */ + list = gst_harness_pull_list (h); + fail_unless (list == NULL); + + /* Pull the buffer at the front of the queue */ + buffer = gst_harness_pull (h); + fail_unless_equals_int (gst_buffer_get_size (buffer), 384); + fail_unless (buffer != NULL); + gst_buffer_unref (buffer); + + /* Pull the list that is now at the front of the queue */ + list = gst_harness_pull_list (h); + fail_unless (list != NULL); + fail_unless_equals_int (gst_buffer_list_length (list), 4); + for (int i = 0; i < 2; ++i) { + buffer = gst_buffer_list_get(list, i); + fail_unless_equals_int(gst_buffer_get_size(buffer), i*937); + } + gst_buffer_list_unref (list); + + gst_harness_teardown (h); +} + +GST_END_TEST; + + static Suite * gst_harness_suite (void) { @@ -304,6 +403,7 @@ gst_harness_suite (void) tcase_add_test (tc_chain, test_harness_empty); tcase_add_test (tc_chain, test_harness_element_ref); tcase_add_test (tc_chain, test_src_harness); + tcase_add_test (tc_chain, test_src_harness_buflist); tcase_add_test (tc_chain, test_src_harness_no_forwarding); tcase_add_test (tc_chain, test_add_sink_harness_without_sinkpad); From d43b460ba69fc0794c677b7226e7f18f3f43ce2e Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Fri, 18 Mar 2022 13:54:23 +0100 Subject: [PATCH 222/339] gstreamer/harness: make pull timesout configurable Although 60 seconds is often enough, we have seen cases where the harnessed pipeline could taken longer than that to process a buffer. This is especially true when we run under additional tools like Valgrind or Debug builds. --- .../gstreamer/libs/gst/check/gstharness.c | 44 ++++++++++++++++--- .../gstreamer/libs/gst/check/gstharness.h | 3 ++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/check/gstharness.c b/subprojects/gstreamer/libs/gst/check/gstharness.c index 7d2d5f08f3b..9076eca02a5 100644 --- a/subprojects/gstreamer/libs/gst/check/gstharness.c +++ b/subprojects/gstreamer/libs/gst/check/gstharness.c @@ -216,6 +216,8 @@ struct _GstHarnessPrivate gboolean eos_received; GPtrArray *stress; + + guint64 pull_timeout; }; static GstFlowReturn @@ -809,6 +811,9 @@ gst_harness_new_empty (void) /* we have forwarding on as a default */ gst_harness_set_forwarding (h, TRUE); + /* we default to 60s as pull timeout */ + gst_harness_set_pull_timeout (h, G_USEC_PER_SEC * 60); + return h; } @@ -1697,6 +1702,31 @@ gst_harness_set_forward_pad (GstHarness * h, GstPad * fwdpad) (GstObject *) fwdpad); } +/** + * gst_harness_set_pull_timeout: + * @h: a #GstHarness + * @timeout: timeout in microseconds + * + * Sets the timeout to use on all pull-like methods. This affects src and sink + * harnesses if any. + * + * MT safe. + * + * Since: 1.20 + **/ +void +gst_harness_set_pull_timeout (GstHarness * h, guint64 timeout) +{ + GstHarnessPrivate *priv = h->priv; + priv->pull_timeout = timeout; + + /* propagate timeout to src and sink harnesses, if any */ + if (h->src_harness) + gst_harness_set_pull_timeout (h->src_harness, timeout); + if (h->sink_harness) + gst_harness_set_pull_timeout (h->sink_harness, timeout); +} + /** * gst_harness_create_buffer: * @h: a #GstHarness @@ -1797,8 +1827,8 @@ gst_harness_push_list (GstHarness * h, GstBufferList * buffer_list) * @h: a #GstHarness * * Pulls a #GstBuffer from the #GAsyncQueue on the #GstHarness sinkpad. The pull - * will timeout in 60 seconds. This is the standard way of getting a buffer - * from a harnessed #GstElement. + * will timeout in #GstHarness.pull_timeout seconds. This is the standard way + * of getting a buffer from a harnessed #GstElement. * * MT safe. * @@ -1816,7 +1846,7 @@ gst_harness_pull (GstHarness * h) priv = h->priv; queue_element = GST_MINI_OBJECT_CAST (g_async_queue_timeout_pop (priv->buffer_queue, - G_USEC_PER_SEC * 60)); + priv->pull_timeout)); g_mutex_lock (&priv->buf_or_eos_mutex); buf = gst_harness_extract_one_buffer_locked (h, queue_element); g_mutex_unlock (&priv->buf_or_eos_mutex); @@ -2337,7 +2367,7 @@ gst_harness_push_event (GstHarness * h, GstEvent * event) * @h: a #GstHarness * * Pulls an #GstEvent from the #GAsyncQueue on the #GstHarness sinkpad. - * Timeouts after 60 seconds similar to gst_harness_pull. + * Timeouts after #GstHarness.pull_timeout seconds similar to gst_harness_pull. * * MT safe. * @@ -2350,7 +2380,7 @@ gst_harness_pull_event (GstHarness * h) { GstHarnessPrivate *priv = h->priv; return (GstEvent *) g_async_queue_timeout_pop (priv->sink_event_queue, - G_USEC_PER_SEC * 60); + priv->pull_timeout); } /** @@ -2440,7 +2470,7 @@ gst_harness_push_upstream_event (GstHarness * h, GstEvent * event) * @h: a #GstHarness * * Pulls an #GstEvent from the #GAsyncQueue on the #GstHarness srcpad. - * Timeouts after 60 seconds similar to gst_harness_pull. + * Timeouts after #GstHarness.pull_timeout seconds similar to gst_harness_pull. * * MT safe. * @@ -2453,7 +2483,7 @@ gst_harness_pull_upstream_event (GstHarness * h) { GstHarnessPrivate *priv = h->priv; return (GstEvent *) g_async_queue_timeout_pop (priv->src_event_queue, - G_USEC_PER_SEC * 60); + priv->pull_timeout); } /** diff --git a/subprojects/gstreamer/libs/gst/check/gstharness.h b/subprojects/gstreamer/libs/gst/check/gstharness.h index 4b310b6f42b..633bd6dc4bc 100644 --- a/subprojects/gstreamer/libs/gst/check/gstharness.h +++ b/subprojects/gstreamer/libs/gst/check/gstharness.h @@ -174,6 +174,9 @@ void gst_harness_set_blocking_push_mode (GstHarness * h); GST_CHECK_API void gst_harness_set_forwarding (GstHarness * h, gboolean forwarding); +GST_CHECK_API +void gst_harness_set_pull_timeout (GstHarness * h, guint64 timeout); + /* buffers */ GST_CHECK_API From 576cbdbb3172b33bc3203ed371014842e40b74d0 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 31 Mar 2022 16:15:38 +1100 Subject: [PATCH 223/339] rtprtxsend: add "limiting" and "stuffing" For "limiting" this means using the token bucket algorithm, allowing a sender to control the bitrate being consumed by rtx-packets, in case of an "out-of-control" receiver asking for way too much packets. 'max-kbps' sets the maximum kilobit per second that the RTX element will give out based on requests, limiting the ability for a receiver to congest the available network bandwidth. 'max-bucket-size' is related to the token bucket algorithm, and will allow control of the "burstiness" of the RTX requests, preventing a huge spike in the network. For "stuffing" this means adding RTX packets as stuffing to meet the total target bitrate 'stuffing-kbps'. There are cases when output rate sent on the network needs to be near constant, but the media isn't (Encoders producing variable bitrates) Also, for probing the network for available bitrate, the stuffing can be used to create redundant information at a certain bitrate. Co-authored-by: Tulio Beloqui Co-authored-by: Stian Selnes --- .../docs/gst_plugins_cache.json | 56 +++ .../gst/rtpmanager/gstrtprtxsend.c | 369 +++++++++++++- .../gst/rtpmanager/gstrtprtxsend.h | 14 + .../gst/rtpmanager/meson.build | 1 + .../gst/rtpmanager/tokenbucket.c | 145 ++++++ .../gst/rtpmanager/tokenbucket.h | 43 ++ .../tests/check/elements/rtprtx.c | 459 +++++++++++++++++- 7 files changed, 1068 insertions(+), 19 deletions(-) create mode 100644 subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.c create mode 100644 subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.h diff --git a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json index bf5ad6657b6..e76bbd5264d 100644 --- a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json +++ b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json @@ -19339,6 +19339,20 @@ "readable": false, "type": "GstStructure", "writable": true + }, + "stuffing-kbps": { + "blurb": "Use RTX to send stuffing packets to produce a near constant rate of total packets (media and stuffing) (0 = disabled)", + "conditionally-available": false, + "construct": true, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "2147483647", + "min": "0", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true } }, "rank": "none", @@ -19398,6 +19412,34 @@ "type": "GstStructure", "writable": true }, + "max-bucket-size": { + "blurb": "The size of the token bucket, related to burstiness resilience (-1 = unlimited)", + "conditionally-available": false, + "construct": true, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "2147483647", + "min": "-1", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + }, + "max-kbps": { + "blurb": "The maximum number of kilobits of RTX packets to allow (-1 = unlimited)", + "conditionally-available": false, + "construct": true, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "2147483647", + "min": "-1", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + }, "max-size-packets": { "blurb": "Amount of packets to queue (0 = unlimited)", "conditionally-available": false, @@ -19475,6 +19517,20 @@ "readable": false, "type": "GstStructure", "writable": true + }, + "stuffing-kbps": { + "blurb": "Use RTX to send stuffing packets to produce a near constant rate of total packets (media and stuffing) (0 = disabled)", + "conditionally-available": false, + "construct": true, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "2147483647", + "min": "0", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true } }, "rank": "none", diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c index 1c36e5272f2..3298a8d4cdc 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c @@ -49,9 +49,16 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_rtx_send_debug); #define GST_CAT_DEFAULT gst_rtp_rtx_send_debug +#define MIN_RTP_HEADER_LEN 12 + +#define UNLIMITED_KBPS (-1) + #define DEFAULT_RTX_PAYLOAD_TYPE 0 #define DEFAULT_MAX_SIZE_TIME 0 #define DEFAULT_MAX_SIZE_PACKETS 100 +#define DEFAULT_MAX_KBPS UNLIMITED_KBPS +#define DEFAULT_MAX_BUCKET_SIZE UNLIMITED_KBPS +#define DEFAULT_STUFFING_KBPS 0 enum { @@ -63,6 +70,10 @@ enum PROP_NUM_RTX_REQUESTS, PROP_NUM_RTX_PACKETS, PROP_CLOCK_RATE_MAP, + PROP_MAX_KBPS, + PROP_MAX_BUCKET_SIZE, + PROP_STUFFING_KBPS, + PROP_LAST, }; enum @@ -156,7 +167,11 @@ G_DEFINE_TYPE_WITH_CODE (GstRtpRtxSend, gst_rtp_rtx_send, GST_TYPE_ELEMENT, GST_ELEMENT_REGISTER_DEFINE (rtprtxsend, "rtprtxsend", GST_RANK_NONE, GST_TYPE_RTP_RTX_SEND); +static void gst_rtp_rtx_send_reset_stuffing_bucket (GstRtpRtxSend * rtx, + gint kbps); + #define IS_RTX_ENABLED(rtx) (g_hash_table_size ((rtx)->rtx_pt_map) > 0) +#define RTX_OVERHEAD 2 typedef struct { @@ -174,6 +189,7 @@ buffer_queue_item_free (BufferQueueItem * item) typedef struct { + guint32 ssrc; guint32 rtx_ssrc; guint16 seqnum_base, next_seqnum; gint clock_rate; @@ -183,10 +199,11 @@ typedef struct } SSRCRtxData; static SSRCRtxData * -ssrc_rtx_data_new (guint32 rtx_ssrc) +ssrc_rtx_data_new (guint32 ssrc, guint32 rtx_ssrc) { SSRCRtxData *data = g_new0 (SSRCRtxData, 1); + data->ssrc = ssrc; data->rtx_ssrc = rtx_ssrc; data->next_seqnum = data->seqnum_base = g_random_int_range (0, G_MAXUINT16); data->queue = g_sequence_new ((GDestroyNotify) buffer_queue_item_free); @@ -337,6 +354,47 @@ gst_rtp_rtx_send_class_init (GstRtpRtxSendClass * klass) G_CALLBACK (gst_rtp_rtx_send_clear_extensions), NULL, NULL, NULL, G_TYPE_NONE, 0); + /** + * rtprtxsend:max-kbps: + * + * The maximum number of kilobits of RTX packets to allow. + * + * Since: 1.22 + */ + g_object_class_install_property (gobject_class, PROP_MAX_KBPS, + g_param_spec_int ("max-kbps", "Maximum Kbps", + "The maximum number of kilobits of RTX packets to allow " + "(-1 = unlimited)", -1, G_MAXINT, DEFAULT_MAX_KBPS, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + /** + * rtprtxsend:max-bucket-size: + * + * The size of the token bucket, related to burstiness resilience. + * + * Since: 1.22 + */ + g_object_class_install_property (gobject_class, PROP_MAX_BUCKET_SIZE, + g_param_spec_int ("max-bucket-size", "Maximum Bucket Size (Kb)", + "The size of the token bucket, related to burstiness resilience " + "(-1 = unlimited)", -1, G_MAXINT, DEFAULT_MAX_BUCKET_SIZE, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + /** + * rtprtxsend:stuffing-kbps: + * + * Use RTX to send stuffing packets to produce a near constant rate of + * total packets (media and stuffing) + * + * Since: 1.22 + */ + g_object_class_install_property (gobject_class, PROP_STUFFING_KBPS, + g_param_spec_int ("stuffing-kbps", "Stuffing kbps", + "Use RTX to send stuffing packets to produce a near constant rate of " + "total packets (media and stuffing) (0 = disabled)", + 0, G_MAXINT, DEFAULT_STUFFING_KBPS, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + gst_element_class_add_static_pad_template (gstelement_class, &src_factory); gst_element_class_add_static_pad_template (gstelement_class, &sink_factory); @@ -427,6 +485,8 @@ gst_rtp_rtx_send_init (GstRtpRtxSend * rtx) rtx->max_size_packets = DEFAULT_MAX_SIZE_PACKETS; rtx->dummy_writable = gst_buffer_new (); + gst_rtp_rtx_send_reset_stuffing_bucket (rtx, DEFAULT_STUFFING_KBPS); + rtx->last_stuffing_ssrc = -1; } static gboolean @@ -458,6 +518,8 @@ gst_rtp_rtx_send_push_out (GstRtpRtxSend * rtx, gpointer object) data->visible = TRUE; data->destroy = gst_rtp_rtx_data_queue_item_free; + GST_LOG_OBJECT (rtx, "Adding %" GST_PTR_FORMAT, object); + success = gst_data_queue_push (rtx->queue, data); if (!success) @@ -498,7 +560,7 @@ gst_rtp_rtx_send_get_ssrc_data (GstRtpRtxSend * rtx, guint32 ssrc) g_free (ssrc_str); } rtx_ssrc = gst_rtp_rtx_send_choose_ssrc (rtx, rtx_ssrc, consider); - data = ssrc_rtx_data_new (rtx_ssrc); + data = ssrc_rtx_data_new (ssrc, rtx_ssrc); g_hash_table_insert (rtx->ssrc_data, GUINT_TO_POINTER (ssrc), data); g_hash_table_insert (rtx->rtx_ssrcs, GUINT_TO_POINTER (rtx_ssrc), GUINT_TO_POINTER (ssrc)); @@ -668,7 +730,7 @@ rewrite_header_extensions (GstRtpRtxSend * rtx, GstRTPBuffer * rtp) * Copy memory to avoid to manually copy each rtp buffer field. */ static GstBuffer * -gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer) +gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer, guint8 padlen) { GstMemory *mem = NULL; GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; @@ -719,6 +781,15 @@ gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer) gst_memory_unmap (mem, &map); gst_buffer_append_memory (new_buffer, mem); + if (padlen) { + mem = gst_allocator_alloc (NULL, padlen, NULL); + gst_memory_map (mem, &map, GST_MAP_WRITE); + memset (map.data, 0, padlen - 1); + map.data[padlen - 1] = padlen; + gst_memory_unmap (mem, &map); + gst_buffer_append_memory (new_buffer, mem); + } + /* everything needed is copied */ gst_rtp_buffer_unmap (&rtp); @@ -728,7 +799,7 @@ gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer) gst_rtp_buffer_set_seq (&new_rtp, seqnum); gst_rtp_buffer_set_payload_type (&new_rtp, fmtp); /* RFC 4588: let other elements do the padding, as normal */ - gst_rtp_buffer_set_padding (&new_rtp, FALSE); + gst_rtp_buffer_set_padding (&new_rtp, padlen != 0); gst_rtp_buffer_unmap (&new_rtp); /* Copy over timestamps */ @@ -750,6 +821,28 @@ buffer_queue_items_cmp (BufferQueueItem * a, BufferQueueItem * b, return gst_rtp_buffer_compare_seqnum (b->seqnum, a->seqnum); } +static gboolean +gst_rtp_rtx_send_token_bucket (GstRtpRtxSend * rtx, GstBuffer * buf) +{ + GstClock *clock; + guint64 tokens; + + /* with an unlimited bucket-size, we have nothing to do */ + if (rtx->max_bucket_size == UNLIMITED_KBPS || rtx->max_kbps == UNLIMITED_KBPS) + return TRUE; + + /* without a clock, nothing to do */ + clock = GST_ELEMENT_CAST (rtx)->clock; + if (clock == NULL) { + GST_WARNING_OBJECT (rtx, "No clock, can't get the time"); + return TRUE; + } + + tokens = gst_buffer_get_size (buf) * 8; + token_bucket_add_tokens (&rtx->max_tb, gst_clock_get_time (clock)); + return token_bucket_take_tokens (&rtx->max_tb, tokens, FALSE); +} + static gboolean gst_rtp_rtx_send_src_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -795,8 +888,13 @@ gst_rtp_rtx_send_src_event (GstPad * pad, GstObject * parent, GstEvent * event) (GCompareDataFunc) buffer_queue_items_cmp, NULL); if (iter) { BufferQueueItem *item = g_sequence_get (iter); - GST_LOG_OBJECT (rtx, "found %u", item->seqnum); - rtx_buf = gst_rtp_rtx_buffer_new (rtx, item->buffer); + GST_LOG_OBJECT (rtx, "found %" G_GUINT16_FORMAT, item->seqnum); + if (gst_rtp_rtx_send_token_bucket (rtx, item->buffer)) { + rtx_buf = gst_rtp_rtx_buffer_new (rtx, item->buffer, 0); + } else { + GST_DEBUG_OBJECT (rtx, "Packet #%" G_GUINT16_FORMAT + " dropped due to full bucket", item->seqnum); + } } #ifndef GST_DISABLE_DEBUG else { @@ -1010,12 +1108,12 @@ gst_rtp_rtx_send_get_ts_diff (SSRCRtxData * data) } /* Must be called with lock */ -static void +static SSRCRtxData * process_buffer (GstRtpRtxSend * rtx, GstBuffer * buffer) { GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; BufferQueueItem *item; - SSRCRtxData *data; + SSRCRtxData *data = NULL; guint16 seqnum; guint8 payload_type; guint32 ssrc, rtptime; @@ -1058,19 +1156,180 @@ process_buffer (GstRtpRtxSend * rtx, GstBuffer * buffer) g_sequence_remove (g_sequence_get_begin_iter (data->queue)); } } + + return data; +} + +static gint +get_buffer_payload_len (GstBuffer * buffer) +{ + gint length = gst_buffer_get_size (buffer) - MIN_RTP_HEADER_LEN; + + if (GST_BUFFER_FLAG_IS_SET (buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION)) { + length -= RTX_OVERHEAD; + } + + return length; +} + +static GstFlowReturn +gst_rtp_rtx_send_push (GstRtpRtxSend * rtx, GstBuffer * buffer) +{ + GstFlowReturn ret; + token_bucket_take_tokens (&rtx->stuff_tb, + get_buffer_payload_len (buffer) * 8, TRUE); + + GST_OBJECT_UNLOCK (rtx); + ret = gst_pad_push (rtx->srcpad, buffer); + GST_OBJECT_LOCK (rtx); + + return ret; +} + +static GSequenceIter * +gst_rtp_rtx_send_get_initial_stuffing_buffer (GstRtpRtxSend * rtx, + SSRCRtxData * rtx_data, gboolean check_window) +{ + GstClockTime running_time; + GstClockTime window_size = 100 * GST_MSECOND; + + GSequenceIter *first = NULL; + GSequenceIter *last = NULL; + + gint available_stuffing_bits = 0; + gint64 bucket_size = rtx->stuff_tb.bucket_size; + + /* determine the first and last item on the queue to do stuffing with */ + last = g_sequence_get_end_iter (rtx_data->queue); + first = last; + + running_time = + gst_clock_get_time (GST_ELEMENT_CLOCK (rtx)) - + GST_ELEMENT_CAST (rtx)->base_time; + + do { + BufferQueueItem *item = g_sequence_get (g_sequence_iter_prev (first)); + gint buffer_bitsize = get_buffer_payload_len (item->buffer) * 8; + + /* stop here if we will exceed the bucket with this buffer */ + if (available_stuffing_bits + buffer_bitsize > bucket_size) + break; + + /* stop here if the packet is "too old" */ + if (check_window && GST_BUFFER_PTS_IS_VALID (item->buffer) + && ABS (GST_CLOCK_DIFF (GST_BUFFER_PTS (item->buffer), + running_time)) > window_size) { + break; + } + + available_stuffing_bits += buffer_bitsize; + first = g_sequence_iter_prev (first); + } while (first != g_sequence_get_begin_iter (rtx_data->queue)); + + if (available_stuffing_bits) + return first; + + /* not enough bits for stuffing */ + return NULL; +} + + +static GstFlowReturn +gst_rtp_rtx_send_push_stuffing (GstRtpRtxSend * rtx, SSRCRtxData * rtx_data) +{ + GstFlowReturn ret = GST_FLOW_OK; + + GSequenceIter *first = NULL; + GSequenceIter *last = NULL; + + gint64 bucket_size = rtx->stuff_tb.bucket_size; + + if (bucket_size <= 0) + return GST_FLOW_OK; + + /* double check the queue */ + if (g_sequence_is_empty (rtx_data->queue)) + return GST_FLOW_OK; + + last = g_sequence_get_end_iter (rtx_data->queue); + + /* check for the first buffer under the window */ + first = gst_rtp_rtx_send_get_initial_stuffing_buffer (rtx, rtx_data, TRUE); + if (!first) { + /* if there is none, fallback to use the last ones */ + first = gst_rtp_rtx_send_get_initial_stuffing_buffer (rtx, rtx_data, FALSE); + } + + /* we cannot generate any stuffing, no bits */ + if (!first) + return GST_FLOW_OK; + + while (first != last) { + BufferQueueItem *item = g_sequence_get (first); + GstBuffer *rtx_buf = gst_rtp_rtx_buffer_new (rtx, item->buffer, 0); + + GST_DEBUG_OBJECT (rtx, "Produce 1 stuffing packets with ssrc %X, " + "original buffer size %" G_GSIZE_FORMAT, rtx_data->rtx_ssrc, + gst_buffer_get_size (item->buffer)); + + ret = gst_rtp_rtx_send_push (rtx, rtx_buf); + first = g_sequence_iter_next (first); + } + + bucket_size = rtx->stuff_tb.bucket_size; + /* we still have some left over budget, call this once more */ + if (bucket_size > 0 && ret == GST_FLOW_OK) + return gst_rtp_rtx_send_push_stuffing (rtx, rtx_data); + + return ret; +} + +/* call with LOCK */ +static SSRCRtxData * +gst_rtp_rtx_send_get_rtx_data (GstRtpRtxSend * rtx, GstBuffer * buffer) +{ + SSRCRtxData *rtx_data = process_buffer (rtx, buffer); + + if (rtx_data) { + /* we have rtx data from the current buffer, save the ssrc */ + rtx->last_stuffing_ssrc = rtx_data->ssrc; + return rtx_data; + } + + if (rtx->last_stuffing_ssrc != -1) { + /* we do not have rtx data from the current buffer, so fetch the last one sent */ + return gst_rtp_rtx_send_get_ssrc_data (rtx, rtx->last_stuffing_ssrc); + } + + return NULL; } static GstFlowReturn gst_rtp_rtx_send_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) { - GstRtpRtxSend *rtx = GST_RTP_RTX_SEND_CAST (parent); GstFlowReturn ret; + GstRtpRtxSend *rtx; + SSRCRtxData *rtx_data = NULL; + GstClock *clock; + + rtx = GST_RTP_RTX_SEND_CAST (parent); GST_OBJECT_LOCK (rtx); - if (rtx->rtx_pt_map_structure) - process_buffer (rtx, buffer); + clock = GST_ELEMENT_CLOCK (rtx); + + if (clock) { + token_bucket_add_tokens (&rtx->stuff_tb, gst_clock_get_time (clock)); + } + + if (IS_RTX_ENABLED (rtx)) + rtx_data = gst_rtp_rtx_send_get_rtx_data (rtx, buffer); + + ret = gst_rtp_rtx_send_push (rtx, buffer); + + if (ret == GST_FLOW_OK && clock && rtx_data && rtx->stuffing_kbps != 0) + ret = gst_rtp_rtx_send_push_stuffing (rtx, rtx_data); + GST_OBJECT_UNLOCK (rtx); - ret = gst_pad_push (rtx->srcpad, buffer); return ret; } @@ -1090,7 +1349,16 @@ gst_rtp_rtx_send_chain_list (GstPad * pad, GstObject * parent, GstFlowReturn ret; GST_OBJECT_LOCK (rtx); - gst_buffer_list_foreach (list, process_buffer_from_list, rtx); + if (IS_RTX_ENABLED (rtx)) { + if (rtx->stuffing_kbps != 0) + GST_WARNING_OBJECT (rtx, "RTX stuffing for chain_list not implemented"); + + gst_buffer_list_foreach (list, process_buffer_from_list, rtx); + + GST_OBJECT_UNLOCK (rtx); + gst_rtp_rtx_send_push_out (rtx, list); + return GST_FLOW_OK; + } GST_OBJECT_UNLOCK (rtx); ret = gst_pad_push_list (rtx->srcpad, list); @@ -1110,9 +1378,9 @@ gst_rtp_rtx_send_src_loop (GstRtpRtxSend * rtx) GST_OBJECT_LOCK (rtx); /* Update statistics just before pushing. */ rtx->num_rtx_packets++; + gst_rtp_rtx_send_push (rtx, GST_BUFFER (data->object)); GST_OBJECT_UNLOCK (rtx); - gst_pad_push (rtx->srcpad, GST_BUFFER (data->object)); } else if (GST_IS_EVENT (data->object)) { gst_pad_push_event (rtx->srcpad, GST_EVENT (data->object)); @@ -1192,6 +1460,21 @@ gst_rtp_rtx_send_get_property (GObject * object, g_value_set_boxed (value, rtx->clock_rate_map_structure); GST_OBJECT_UNLOCK (rtx); break; + case PROP_MAX_KBPS: + GST_OBJECT_LOCK (rtx); + g_value_set_int (value, rtx->max_kbps); + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_MAX_BUCKET_SIZE: + GST_OBJECT_LOCK (rtx); + g_value_set_int (value, rtx->max_bucket_size); + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_STUFFING_KBPS: + GST_OBJECT_LOCK (rtx); + g_value_set_int (value, rtx->stuffing_kbps); + GST_OBJECT_UNLOCK (rtx); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1214,6 +1497,36 @@ structure_to_hash_table (GQuark field_id, const GValue * value, gpointer hash) return TRUE; } +static void +gst_rtp_rtx_send_reset_max_bucket (GstRtpRtxSend * rtx, + gint max_kbps, gint max_bucket_size) +{ + gboolean prev_unlimited = rtx->max_kbps == UNLIMITED_KBPS || + rtx->max_bucket_size == UNLIMITED_KBPS; + gboolean unlimited = max_kbps == UNLIMITED_KBPS || + max_bucket_size == UNLIMITED_KBPS; + + rtx->max_kbps = max_kbps; + rtx->max_bucket_size = max_bucket_size; + + token_bucket_init (&rtx->max_tb, + max_kbps == -1 ? -1 : max_kbps * 1000, + max_bucket_size == -1 ? -1 : max_bucket_size * 1000); + + /* Fill the bucket to max if we switched from unlimited to limited + * kbps mode */ + if (prev_unlimited && !unlimited) { + rtx->max_tb.bucket_size = max_bucket_size * 1000; + } +} + +static void +gst_rtp_rtx_send_reset_stuffing_bucket (GstRtpRtxSend * rtx, gint kbps) +{ + rtx->stuffing_kbps = kbps; + token_bucket_init (&rtx->stuff_tb, kbps * 1000, kbps * 1000); +} + static void gst_rtp_rtx_send_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) @@ -1234,8 +1547,8 @@ gst_rtp_rtx_send_set_property (GObject * object, gst_structure_free (rtx->rtx_pt_map_structure); rtx->rtx_pt_map_structure = g_value_dup_boxed (value); g_hash_table_remove_all (rtx->rtx_pt_map); - gst_structure_foreach (rtx->rtx_pt_map_structure, structure_to_hash_table, - rtx->rtx_pt_map); + gst_structure_foreach (rtx->rtx_pt_map_structure, + structure_to_hash_table, rtx->rtx_pt_map); GST_OBJECT_UNLOCK (rtx); if (IS_RTX_ENABLED (rtx)) @@ -1264,6 +1577,30 @@ gst_rtp_rtx_send_set_property (GObject * object, structure_to_hash_table, rtx->clock_rate_map); GST_OBJECT_UNLOCK (rtx); break; + case PROP_MAX_KBPS: + GST_OBJECT_LOCK (rtx); + { + gint max_kbps = g_value_get_int (value); + gst_rtp_rtx_send_reset_max_bucket (rtx, max_kbps, rtx->max_bucket_size); + } + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_MAX_BUCKET_SIZE: + GST_OBJECT_LOCK (rtx); + { + gint max_bucket_size = g_value_get_int (value); + gst_rtp_rtx_send_reset_max_bucket (rtx, rtx->max_kbps, max_bucket_size); + } + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_STUFFING_KBPS: + GST_OBJECT_LOCK (rtx); + { + gint kbps = g_value_get_int (value); + gst_rtp_rtx_send_reset_stuffing_bucket (rtx, kbps); + } + GST_OBJECT_UNLOCK (rtx); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h index 60a4ec5bb91..42a24719615 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h @@ -27,6 +27,7 @@ #include #include #include +#include "tokenbucket.h" G_BEGIN_DECLS @@ -83,6 +84,19 @@ struct _GstRtpRtxSend GstRTPHeaderExtension *rid_repaired; GstBuffer *dummy_writable; + + /* bucket */ + gint max_kbps; + gint max_bucket_size; + TokenBucket max_tb; + + /* stuffing properties */ + gint stuffing_kbps; + + /* last ssrc used for stuffing */ + gint last_stuffing_ssrc; + + TokenBucket stuff_tb; }; struct _GstRtpRtxSendClass diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/meson.build b/subprojects/gst-plugins-good/gst/rtpmanager/meson.build index 20c95f742c9..0a0e6a14a9e 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/meson.build +++ b/subprojects/gst-plugins-good/gst/rtpmanager/meson.build @@ -12,6 +12,7 @@ rtpmanager_sources = [ 'gstrtphdrext-roi.c', 'gstrtpmux.c', 'gstrtpptdemux.c', + 'tokenbucket.c', 'gstrtprtxqueue.c', 'gstrtprtxreceive.c', 'gstrtprtxsend.c', diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.c b/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.c new file mode 100644 index 00000000000..6a4366e64c1 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.c @@ -0,0 +1,145 @@ +/* GStreamer +* Copyright (C) 2021 Pexip (http://pexip.com/) +* @author: Havard Graff +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Library General Public +* License as published by the Free Software Foundation; either +* version 2 of the License, or (at your option) any later version. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Library General Public License for more details. +* +* You should have received a copy of the GNU Library General Public +* License along with this library; if not, write to the +* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +* Boston, MA 02110-1301, USA. +*/ +#include "tokenbucket.h" + +void +token_bucket_reset (TokenBucket * tb) +{ + tb->prev_time = GST_CLOCK_TIME_NONE; + tb->bucket_size = 0; +} + +void +token_bucket_set_bps (TokenBucket * tb, gint64 bps) +{ + tb->bps = bps; +} + +void +token_bucket_set_max_bucket_size (TokenBucket * tb, gint64 max_bucket_size) +{ + tb->max_bucket_size = max_bucket_size; +} + +void +token_bucket_init (TokenBucket * tb, gint64 bps, gint max_bucket_size) +{ + token_bucket_reset (tb); + token_bucket_set_bps (tb, bps); + token_bucket_set_max_bucket_size (tb, max_bucket_size); +} + +void +token_bucket_add_tokens (TokenBucket * tb, GstClockTime now) +{ + gint64 tokens = 0; + GstClockTimeDiff elapsed_time = 0; + + if (!GST_CLOCK_TIME_IS_VALID (now)) + return; + + /* get the elapsed time */ + if (GST_CLOCK_TIME_IS_VALID (tb->prev_time)) { + if (now < tb->prev_time) { + GST_INFO ("We have already produced tokens for this time " + "(%" GST_TIME_FORMAT " < %" GST_TIME_FORMAT ")", + GST_TIME_ARGS (now), GST_TIME_ARGS (tb->prev_time)); + } else { + elapsed_time = GST_CLOCK_DIFF (tb->prev_time, now); + } + } + + tb->prev_time = now; + + /* check for umlimited bps and fill the bucket if that is the case */ + if (tb->bps == -1) { + if (tb->max_bucket_size != -1) + tb->bucket_size = tb->max_bucket_size; + return; + } + + /* no bitrate, no bits to add */ + if (tb->bps == 0) { + return; + } + + /* no extra time to add tokens for */ + if (elapsed_time == 0) { + return; + } + + /* calculate number of tokens and add them to the bucket */ + tokens = gst_util_uint64_scale_round (elapsed_time, tb->bps, GST_SECOND); + tb->bucket_size += tokens; + if (tb->max_bucket_size != -1 && tb->bucket_size > tb->max_bucket_size) { + tb->bucket_size = tb->max_bucket_size; + } + GST_LOG ("Added %" G_GINT64_FORMAT " tokens to bucket " + "(contains %" G_GINT64_FORMAT " tokens)", tokens, tb->bucket_size); + + GST_LOG ("Elapsed time: %" GST_TIME_FORMAT " produces %" G_GINT64_FORMAT + " tokens, new prev_time: %" GST_TIME_FORMAT, + GST_TIME_ARGS (elapsed_time), tokens, GST_TIME_ARGS (tb->prev_time)); +} + +GstClockTime +token_bucket_get_missing_tokens_time (TokenBucket * tb, gint tokens) +{ + GstClockTimeDiff ret; + gint missing_tokens; + + /* unlimited bits per second and unlimited bucket size + means tokens are always available */ + if (tb->bps == -1 && tb->max_bucket_size == -1) + return 0; + + /* we we have room in our bucket, no need to wait */ + if (tb->bucket_size >= tokens) + return 0; + + /* lets not divide by 0 */ + if (tb->bps == 0) + return 0; + + missing_tokens = tokens - tb->bucket_size; + ret = gst_util_uint64_scale (GST_SECOND, missing_tokens, tb->bps); + + return ret; +} + +gboolean +token_bucket_take_tokens (TokenBucket * tb, gint tokens, gboolean force) +{ + /* unlimited bits per second and unlimited bucket size + means tokens are always available */ + if (tb->bps == -1 && tb->max_bucket_size == -1) + return TRUE; + + /* if we force it, or we have enough tokens, remove + the tokens from the bucket */ + if (force || tb->bucket_size >= tokens) { + GST_LOG ("Removing %u tokens from bucket (%" G_GINT64_FORMAT ")", + tokens, tb->bucket_size); + tb->bucket_size -= tokens; + return TRUE; + } + + return FALSE; +} diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.h b/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.h new file mode 100644 index 00000000000..2178c245690 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.h @@ -0,0 +1,43 @@ +/* GStreamer +* Copyright (C) 2021 Pexip (http://pexip.com/) +* @author: Havard Graff +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Library General Public +* License as published by the Free Software Foundation; either +* version 2 of the License, or (at your option) any later version. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Library General Public License for more details. +* +* You should have received a copy of the GNU Library General Public +* License along with this library; if not, write to the +* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +* Boston, MA 02110-1301, USA. +*/ +#ifndef __TOKEN_BUCKET_H__ +#define __TOKEN_BUCKET_H__ + +#include + +typedef struct _TokenBucket TokenBucket; + +struct _TokenBucket +{ + gint64 max_bucket_size; + gint64 bps; + GstClockTime prev_time; + gint64 bucket_size; +}; + +void token_bucket_init (TokenBucket * tb, gint64 bps, gint max_bucket_size); +void token_bucket_set_bps (TokenBucket * tb, gint64 bps); +void token_bucket_set_max_bucket_size (TokenBucket * tb, gint64 bps); +void token_bucket_reset (TokenBucket * tb); +void token_bucket_add_tokens (TokenBucket * tb, GstClockTime now); +GstClockTime token_bucket_get_missing_tokens_time (TokenBucket * tb, gint tokens); +gboolean token_bucket_take_tokens (TokenBucket * tb, gint tokens, gboolean force); + +#endif /* __TOKEN_BUCKET_H__ */ diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c index 1580c8bf4a9..13bd198266d 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c @@ -131,9 +131,9 @@ create_rtp_buffer_ex (guint32 ssrc, guint8 payload_type, guint16 seqnum, } static GstBuffer * -create_rtp_buffer (guint32 ssrc, guint8 payload_type, guint16 seqnum) +create_rtp_buffer_with_payload_size (guint32 ssrc, guint8 payload_type, + guint16 seqnum, guint payload_size) { - guint payload_size = 29; guint64 timestamp = gst_util_uint64_scale_int (seqnum, 90000, 30); GstRTPBuffer *rtpbuf = create_rtp_buffer_ex (ssrc, payload_type, seqnum, (guint32) timestamp, payload_size); @@ -146,6 +146,12 @@ create_rtp_buffer (guint32 ssrc, guint8 payload_type, guint16 seqnum) return ret; } +static GstBuffer * +create_rtp_buffer (guint32 ssrc, guint8 payload_type, guint16 seqnum) +{ + return create_rtp_buffer_with_payload_size (ssrc, payload_type, seqnum, 29); +} + static GstBuffer * create_rtp_buffer_with_timestamp (guint32 ssrc, guint8 payload_type, guint16 seqnum, guint32 timestamp, GstClockTime pts) @@ -1052,7 +1058,7 @@ GST_START_TEST (test_rtxsend_header_extensions) GstRTPHeaderExtension *send_stream_id, *send_repaired_stream_id; GstRTPHeaderExtension *recv_stream_id, *recv_repaired_stream_id; guint stream_hdr_id = 1, repaired_hdr_id = 2; - gint i; + guint i; pt_map = gst_structure_new ("application/x-rtp-pt-map", "96", G_TYPE_UINT, rtx_pt, NULL); @@ -1162,6 +1168,444 @@ GST_START_TEST (test_rtxsend_header_extensions) GST_END_TEST; +GST_START_TEST (test_rtxsender_stuffing) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* 800 kbps = 100 kBps. 10 ms distance between each packet means 100 packets + * per second and 1000 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + gst_harness_set_time (h, 10 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 580)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + /* budget 1000, sent 600, no stuffing (stuff with 580 will exceed budget) */ + + gst_harness_set_time (h, 20 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 600)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + /* budget 2000, sent 1200, stuff with #2, 600 bytes */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + /* budget 2000, sent 1800, no stuffing */ + + gst_harness_set_time (h, 30 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 3, 200)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 3); + /* budget 3000, sent 2000. Stuffed packets will be of (600+200+200)=1000 bytes */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); + /* budget 3000, sent 3000 */ + + gst_harness_set_time (h, 40 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 4, 600)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 4); + /* budget 4000, sent 3600, no stuffing because rtx packet is 600 */ + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +/* +* This tests verify that the element doesn't generate stuffing buffers, unless +* the property is set to non zero. +*/ +GST_START_TEST (test_rtxsender_stuffing_toggle) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 0, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + gst_harness_set_time (h, 10 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + + gst_harness_set_time (h, 20 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + + /* 80 kbps = 10 kBps. 10 ms distance between each packet means 10 packets + * per second and 100 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + g_object_set (h->element, "stuffing-kbps", 80, NULL); + + gst_harness_set_time (h, 30 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 3, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 3); + + gst_harness_set_time (h, 40 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 4, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 4); + + /* budget 100, sent 40, so stuff with #2, 3 and 4, total of 60 bytes */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 4); + + /* disable stuffing */ + g_object_set (h->element, "stuffing-kbps", 0, NULL); + + gst_harness_set_time (h, 50 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 5, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 5); + + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +/* +* This tests verify that buffers with payload types that are not marked for +* RTX, also generate stuffing. +*/ +GST_START_TEST (test_rtxsender_stuffing_non_rtx_packets) +{ + guint video_ssrc = 1234567; + guint video_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + guint audio_ssrc = 6789123; + guint audio_pt = 111; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* 800 kbps = 100 kBps. 10 ms distance between each packet means 100 packets + * per second and 1000 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 0, 20)); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 0); + + gst_harness_set_time (h, 10 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 1, 580)); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 1); + /* budget 1000, sent 600, no stuffing (stuff with 580 will exceed budget) */ + + gst_harness_set_time (h, 20 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 2, 600)); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 2); + /* budget 2000, sent 1200, stuff with #2, 600 bytes */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + + gst_harness_set_time (h, 30 * GST_MSECOND); + /* audio packet comes in.. */ + gst_harness_push (h, + create_rtp_buffer_with_payload_size (audio_ssrc, audio_pt, 0, 200)); + pull_and_verify (h, FALSE, audio_ssrc, audio_pt, 0); + /* budget 3000, sent 2000, stuff with #2, 600 bytes */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + + gst_harness_push (h, + create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 4, 600)); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 4); + /* budget 4000, sent 2600, no more stuffing */ + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +/* +* This tests verify that buffers over the window size time are not included for stuffing. +*/ +GST_START_TEST (test_rtxsender_stuffing_window) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + GstBuffer *buf; + gint i; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 80, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* 80 kbps = 10 kBps. 10 ms distance between each packet means 10 packets + * per second and 100 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + gst_harness_set_time (h, 0 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 80); + GST_BUFFER_PTS (buf) = 0; + gst_harness_push (h, buf); + + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + /* budget 100, sent 80 */ + + gst_harness_set_time (h, 10 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 80); + GST_BUFFER_PTS (buf) = 10 * GST_MSECOND; + gst_harness_push (h, buf); + + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + /* budget 200, sent 160, no stuffing */ + + gst_harness_set_time (h, 110 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 80); + GST_BUFFER_PTS (buf) = 110 * GST_MSECOND; + gst_harness_push (h, buf); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + + /* we have budget, so stuff with #1 and #2, but #0 is too old */ + for (i = 0; i < 5; i++) { + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + } + + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +/* +* This tests verify that even though there is no packets under the stuffing +* window, the element still produces stuffing. +*/ +GST_START_TEST (test_rtxsender_stuffing_no_packets_under_window) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + GstBuffer *buf; + gint i; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 80, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* 80 kbps = 10 kBps. 10 ms distance between each packet means 10 packets + * per second and 100 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + gst_harness_set_time (h, 0 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 80); + GST_BUFFER_PTS (buf) = 0; + gst_harness_push (h, buf); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + /* budget 100, sent 80 */ + + gst_harness_set_time (h, 10 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 80); + GST_BUFFER_PTS (buf) = 10 * GST_MSECOND; + gst_harness_push (h, buf); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + /* budget 200, sent 160, no stuffing */ + + /* advance far ahead, so the buffers are out of the window */ + gst_harness_set_time (h, 150 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 80); + GST_BUFFER_PTS (buf) = 20 * GST_MSECOND; + gst_harness_push (h, buf); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + + /* we have budget, non of the above are under the window, but we get stuff with: #0, 1 and 2 */ + for (i = 0; i < 5; i++) { + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 0); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + } + + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtxsender_stuffing_sanity_when_input_rate_is_extreme) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + gint i; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 1, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* Produce an insane amount of data so that variables may overflow (more + * than G_MAXINT bits). No stuffing should be produced. */ + for (i = 0; i < 300; i++) { + gst_harness_set_time (h, i * GST_NSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, i, + 1000000)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, i); + } + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtxsender_stuffing_does_not_interfer_with_rtx) +{ + /* Stuffing packets should be considered a part of the original data in + * terms of counting bits and should not interfer with rtx-kbps */ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, "max-kbps", 400, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 100)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + gst_harness_set_time (h, 10 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 500)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + /* budget 1000, sent 600 */ + + /* Request and send RTX packet. Not calculated in budget */ + gst_harness_push_upstream_event (h, + create_rtx_event (master_ssrc, master_pt, 1)); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + + /* Send media packet. Budget should still be untouched for before. */ + gst_harness_set_time (h, 20 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 400)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + /* budet 2000, sent 1000, send #2: 402 bytes of stuffing */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + static Suite * rtprtx_suite (void) { @@ -1190,6 +1634,15 @@ rtprtx_suite (void) tcase_add_test (tc_chain, test_rtxsend_header_extensions); tcase_add_test (tc_chain, test_rtxsend_header_extensions_copy); + tcase_add_test (tc_chain, test_rtxsender_stuffing); + tcase_add_test (tc_chain, test_rtxsender_stuffing_toggle); + tcase_add_test (tc_chain, test_rtxsender_stuffing_non_rtx_packets); + tcase_add_test (tc_chain, test_rtxsender_stuffing_window); + tcase_add_test (tc_chain, test_rtxsender_stuffing_no_packets_under_window); + tcase_add_test (tc_chain, + test_rtxsender_stuffing_sanity_when_input_rate_is_extreme); + tcase_add_test (tc_chain, test_rtxsender_stuffing_does_not_interfer_with_rtx); + return s; } From 60a443eed82418f238541dc7268362e1de6e0fed Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 6 Apr 2022 12:13:46 -0300 Subject: [PATCH 224/339] rtprtxsend: fixed copying twcc ext-hdr if present --- .../gst/rtpmanager/gstrtprtxsend.c | 7 +- .../tests/check/elements/rtprtx.c | 92 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c index 3298a8d4cdc..02e50f8aa16 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c @@ -766,7 +766,12 @@ gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer, guint8 padlen) /* copy extension if any */ if (rtp.size[1]) { - mem = rewrite_header_extensions (rtx, &rtp); + if (rtx->rid_stream) { + mem = rewrite_header_extensions (rtx, &rtp); + } else { + mem = gst_memory_copy (rtp.map[1].memory, + (guint8 *) rtp.data[1] - rtp.map[1].data, rtp.size[1]); + } gst_buffer_append_memory (new_buffer, mem); } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c index 13bd198266d..f245e3573d1 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c @@ -146,6 +146,55 @@ create_rtp_buffer_with_payload_size (guint32 ssrc, guint8 payload_type, return ret; } +static GstBuffer * +create_rtp_buffer_with_twcc_ext (guint32 ssrc, guint8 payload_type, + guint16 seqnum, guint payload_size, guint8 twcc_ext_id, guint16 twcc_seqnum) +{ + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstBuffer *buf = + create_rtp_buffer_with_payload_size (ssrc, payload_type, seqnum, + payload_size); + + gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp); + + if (twcc_ext_id > 0) { + guint8 twcc_seqnum_be[2]; + GST_WRITE_UINT16_BE (twcc_seqnum_be, twcc_seqnum); + gst_rtp_buffer_add_extension_onebyte_header (&rtp, twcc_ext_id, + twcc_seqnum_be, sizeof (twcc_seqnum_be)); + } + + gst_rtp_buffer_unmap (&rtp); + + return buf; +} + +static gint16 +read_twcc_seqnum (GstBuffer * buf, guint8 twcc_ext_id) +{ + gint16 twcc_seqnum; + gpointer ext_data; + guint ext_size; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + if (!gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp)) + return -1; + + if (!gst_rtp_buffer_get_extension_onebyte_header (&rtp, twcc_ext_id, + 0, &ext_data, &ext_size)) { + gst_rtp_buffer_unmap (&rtp); + return -1; + } + + fail_unless (ext_data != NULL); + fail_unless (ext_size == 2); + + twcc_seqnum = GST_READ_UINT16_BE (ext_data); + gst_rtp_buffer_unmap (&rtp); + + return twcc_seqnum; +} + static GstBuffer * create_rtp_buffer (guint32 ssrc, guint8 payload_type, guint16 seqnum) { @@ -1041,6 +1090,48 @@ GST_START_TEST (test_rtxsend_header_extensions_copy) GST_END_TEST; +GST_START_TEST (test_rtxsender_copy_twcc_exthdr) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstBuffer *rtx_buf; + guint8 twcc_ext_id = 5; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_twcc_ext (master_ssrc, master_pt, 0, 20, + twcc_ext_id, 33)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + gst_harness_push_upstream_event (h, create_rtx_event (master_ssrc, + master_pt, 0)); + + rtx_buf = gst_harness_pull (h); + verify_buf (rtx_buf, TRUE, rtx_ssrc, rtx_pt, 0); + fail_unless (read_twcc_seqnum (rtx_buf, twcc_ext_id) == 33); + gst_buffer_unref (rtx_buf); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; #define RTPHDREXT_STREAM_ID GST_RTP_HDREXT_BASE "sdes:rtp-stream-id" #define RTPHDREXT_REPAIRED_STREAM_ID GST_RTP_HDREXT_BASE "sdes:repaired-rtp-stream-id" @@ -1633,6 +1724,7 @@ rtprtx_suite (void) tcase_add_test (tc_chain, test_rtxsender_clock_rate_map); tcase_add_test (tc_chain, test_rtxsend_header_extensions); tcase_add_test (tc_chain, test_rtxsend_header_extensions_copy); + tcase_add_test (tc_chain, test_rtxsender_copy_twcc_exthdr); tcase_add_test (tc_chain, test_rtxsender_stuffing); tcase_add_test (tc_chain, test_rtxsender_stuffing_toggle); From e63b545d0e4f25f2a05ba1db6db63dd00ae86ce4 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Thu, 28 Oct 2021 17:03:55 +0900 Subject: [PATCH 225/339] [pexhack] rtphdrextroi: tmp support of multiple roi-types Squashed commit of the following: commit c2e3ed18be Author: Camilo Celis Guzman Date: Tue Nov 2 22:30:12 2021 +0900 rtphdrextroi: correct max. number of roi-types commit ac3c6c4599 Author: Camilo Celis Guzman Date: Fri Oct 29 23:00:14 2021 +0900 rtphdrextroi: add test for multiple roi-types commit 1fd0bc8ad2 Author: Camilo Celis Guzman Date: Thu Oct 28 19:42:04 2021 +0900 rtphdrextroi: WIP: multiple roi-types support commit 4b9f2d8036 Author: Camilo Celis Guzman Date: Thu Oct 28 17:19:57 2021 +0900 rtphdrextroi: add test for case of multiple RoI metas with the same roi-type The expected behaviour is that only one of them is payloaded commit b17f4600e2 Author: Camilo Celis Guzman Date: Thu Oct 28 17:03:55 2021 +0900 rtphdrextroi: make roi-type construct only --- .../gst/rtpmanager/gstrtphdrext-roi.c | 276 ++++++++--- .../tests/check/elements/rtphdrext-roi.c | 445 ++++++++++++++++-- 2 files changed, 622 insertions(+), 99 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c index a2d6968a08a..29cc4fc2656 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c @@ -29,27 +29,35 @@ #include "gstrtphdrext-roi.h" #include +GST_DEBUG_CATEGORY_STATIC (rtp_hdrext_roi_debug); +#define GST_CAT_DEFAULT rtp_hdrext_roi_debug + /* * Internal roi-ext-hdr ID used as a safety mechanism, to make sure * that we not only only care about a specific roi_type (as in * GstVideoRegionOfInterestMeta) but also that it has been payloaded * and payloaded using the set of callbacks implemented below. */ -#define ROI_EXTHDR_TYPE 0xFD - #define ROI_HDR_EXT_URI GST_RTP_HDREXT_BASE"TBD:draft-ford-avtcore-roi-extension-00" +#define ROI_EXTHDR_SIZE 11 +#define ROI_MAX_ROI_TYPES 15 /* 0 is unused and we allocate 1 byte for the ID */ + +#define DEFAULT_ROI_TYPE 126 +#define DEFAULT_ROI_ID 8 + enum { PROP_0, - PROP_ROI_TYPE, + PROP_ROI_TYPES, }; struct _GstRTPHeaderExtensionRoi { GstRTPHeaderExtension parent; - guint roi_type; + GHashTable *roi_types; + GHashTable *roi_ids; }; G_DEFINE_TYPE_WITH_CODE (GstRTPHeaderExtensionRoi, @@ -59,14 +67,36 @@ G_DEFINE_TYPE_WITH_CODE (GstRTPHeaderExtensionRoi, GST_ELEMENT_REGISTER_DEFINE (rtphdrextroi, "rtphdrextroi", GST_RANK_MARGINAL, GST_TYPE_RTP_HEADER_EXTENSION_ROI); +static void +gst_rtp_header_extension_roi_serialize_roi_types (GstRTPHeaderExtensionRoi * + self, GValue * value) +{ + GHashTableIter iter; + GValue val = G_VALUE_INIT; + gpointer key; + + GST_DEBUG_OBJECT (self, "Serialize roi-types %p", self->roi_types); + + g_value_init (&val, G_TYPE_UINT); + g_hash_table_iter_init (&iter, self->roi_types); + + while (g_hash_table_iter_next (&iter, &key, NULL)) { + guint roi_type = GPOINTER_TO_UINT (key); + g_value_set_uint (&val, roi_type); + gst_value_array_append_value (value, &val); + } + + g_value_unset (&val); +} + static void gst_rtp_header_extension_roi_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (object); switch (prop_id) { - case PROP_ROI_TYPE: - g_value_set_uint (value, self->roi_type); + case PROP_ROI_TYPES: + gst_rtp_header_extension_roi_serialize_roi_types (self, value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -74,14 +104,101 @@ gst_rtp_header_extension_roi_get_property (GObject * object, } } +static void +_warn_invalid_roi_type (const GValue * val) +{ + GValue str = G_VALUE_INIT; + + g_value_init (&str, G_TYPE_STRING); + g_value_transform (val, &str); + + g_warning ("Invalid roi-type must be an unsigned int got '%s'", + g_value_get_string (&str)); + + g_value_unset (&str); +} + +static void +gst_rtp_header_extension_roi_reset_roi_types (GstRTPHeaderExtensionRoi * self) +{ + GST_DEBUG_OBJECT (self, "Re-set roi-types"); + + if (self->roi_types) + g_hash_table_destroy (self->roi_types); + if (self->roi_ids) + g_hash_table_destroy (self->roi_ids); + self->roi_types = g_hash_table_new (NULL, NULL); + self->roi_ids = g_hash_table_new (NULL, NULL); +} + +static void +gst_rtp_header_extension_roi_set_roi_types (GstRTPHeaderExtensionRoi * self, + const GValue * value) +{ + guint num_roi_types = gst_value_array_get_size (value); + + GST_DEBUG_OBJECT (self, "Set roi-types with %p", value); + + if (num_roi_types == 0) { + GST_INFO_OBJECT (self, "roi-types length is 0. Only the default roi-type " + "of %u will be allowed", DEFAULT_ROI_TYPE); + gst_rtp_header_extension_roi_reset_roi_types (self); + g_hash_table_insert (self->roi_types, + GUINT_TO_POINTER (DEFAULT_ROI_TYPE), GUINT_TO_POINTER (DEFAULT_ROI_ID)); + g_hash_table_insert (self->roi_ids, + GUINT_TO_POINTER (DEFAULT_ROI_ID), GUINT_TO_POINTER (DEFAULT_ROI_TYPE)); + return; + } + + /* verify our maximum number of roi-types supported: + * we can only payload 1 RoI meta per roi-type; and given the a fixed size + * per header extension we allowed a finite set of roi-tyes */ + if (num_roi_types > ROI_MAX_ROI_TYPES) { + g_warning ("Maximum allowed set of roi-types surpassed. " + "Current length=%d max-length=%d", num_roi_types, ROI_MAX_ROI_TYPES); + return; + } + + /* convert given array into 2 hash talbes: a bi-directional mapping of + * roi-type to roi-id and vice versa. */ + gst_rtp_header_extension_roi_reset_roi_types (self); + for (guint i = 0; i < num_roi_types; i++) { + const GValue *val = gst_value_array_get_value (value, i); + guint roi_type = DEFAULT_ROI_TYPE; + guint roi_id = i + 1; /* avoid using 0 as roi-id */ + + if (!G_VALUE_HOLDS_UINT (val)) { + _warn_invalid_roi_type (val); + g_hash_table_destroy (self->roi_types); + g_hash_table_destroy (self->roi_ids); + return; + } + + roi_type = g_value_get_uint (val); + + /* we dont allow duplicates */ + if (g_hash_table_lookup (self->roi_types, GUINT_TO_POINTER (roi_type))) { + g_warning ("Invalid roi-types. No duplicates are allowed"); + g_hash_table_unref (self->roi_types); + g_hash_table_unref (self->roi_ids); + } + + GST_TRACE_OBJECT (self, "Adding %u:%u as roi-type", roi_type, roi_id); + g_hash_table_insert (self->roi_types, + GUINT_TO_POINTER (roi_type), GUINT_TO_POINTER (roi_id)); + g_hash_table_insert (self->roi_ids, + GUINT_TO_POINTER (roi_id), GUINT_TO_POINTER (roi_type)); + } +} + static void gst_rtp_header_extension_roi_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (object); switch (prop_id) { - case PROP_ROI_TYPE: - self->roi_type = g_value_get_uint (value); + case PROP_ROI_TYPES: + gst_rtp_header_extension_roi_set_roi_types (self, value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -92,17 +209,20 @@ gst_rtp_header_extension_roi_set_property (GObject * object, static GstRTPHeaderExtensionFlags gst_rtp_header_extension_roi_get_supported_flags (GstRTPHeaderExtension * ext) { - return GST_RTP_HEADER_EXTENSION_ONE_BYTE; + // GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); + // return g_hash_table_size (self->roi_types) >= 1 ? + // GST_RTP_HEADER_EXTENSION_ONE_BYTE : GST_RTP_HEADER_EXTENSION_TWO_BYTE; + return GST_RTP_HEADER_EXTENSION_TWO_BYTE; } static gsize gst_rtp_header_extension_roi_get_max_size (GstRTPHeaderExtension * ext, const GstBuffer * input_meta) { - return 11; + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); + return g_hash_table_size (self->roi_types) * ROI_EXTHDR_SIZE; } - static gssize gst_rtp_header_extension_roi_write (GstRTPHeaderExtension * ext, const GstBuffer * input_meta, GstRTPHeaderExtensionFlags write_flags, @@ -111,49 +231,62 @@ gst_rtp_header_extension_roi_write (GstRTPHeaderExtension * ext, GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); GstVideoRegionOfInterestMeta *meta; gpointer state = NULL; + guint offset = 0; g_return_val_if_fail (size >= gst_rtp_header_extension_roi_get_max_size (ext, NULL), -1); g_return_val_if_fail (write_flags & gst_rtp_header_extension_roi_get_supported_flags (ext), -1); + gboolean written_roi_ids[ROI_MAX_ROI_TYPES + 1] = { 0 }; while ((meta = (GstVideoRegionOfInterestMeta *) gst_buffer_iterate_meta_filtered ((GstBuffer *) input_meta, &state, GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE))) { GstStructure *extra_param_s; guint num_faces = 0; - /* we only really care to write RoI metas coming from pexfdbin - * so we filter also by GstVideoRegionOfInterestMeta->roi_type */ - if (self->roi_type != meta->roi_type) + /* filter by known roi-types */ + gpointer roi_id_ptr = g_hash_table_lookup (self->roi_types, + GUINT_TO_POINTER (meta->roi_type)); + guint roi_id = GPOINTER_TO_UINT (roi_id_ptr); + GST_TRACE_OBJECT (self, "Got roi-type %u roi-id %u", meta->roi_type, + roi_id); + + if (!roi_id_ptr) continue; - // FIXME: Issue 23027 - // If these asserts got hit the problem either in the scaler of in crop element - // g_assert (meta->w > 0); - // g_assert (meta->h > 0); + if (written_roi_ids[roi_id]) { + GST_TRACE_OBJECT (self, "Already written RoI meta with type %d. " + "Skipping.", meta->roi_type); + continue; + } /* RoI coordinates */ - GST_WRITE_UINT16_BE (&data[0], meta->x); - GST_WRITE_UINT16_BE (&data[2], meta->y); - GST_WRITE_UINT16_BE (&data[4], meta->w); - GST_WRITE_UINT16_BE (&data[6], meta->h); - - /* RoI type (as in the roi-ext-hdr type) */ - GST_WRITE_UINT8 (&data[8], ROI_EXTHDR_TYPE); + GST_WRITE_UINT16_BE (&data[offset + 0], meta->x); + GST_WRITE_UINT16_BE (&data[offset + 2], meta->y); + GST_WRITE_UINT16_BE (&data[offset + 4], meta->w); + GST_WRITE_UINT16_BE (&data[offset + 6], meta->h); - /* RoI - number of faces */ + /* RoI type */ + g_assert (roi_id <= G_MAXUINT8); + GST_WRITE_UINT8 (&data[offset + 8], (guint8) roi_id); + /* RoI extra: number of faces */ extra_param_s = gst_video_region_of_interest_meta_get_param (meta, "extra-param"); if (extra_param_s) gst_structure_get_uint (extra_param_s, "num_faces", &num_faces); - GST_WRITE_UINT16_BE (&data[9], num_faces); + GST_WRITE_UINT16_BE (&data[offset + 9], num_faces); - return 11; + GST_TRACE_OBJECT (self, "Wrote header extension from RoI meta " + "(%u %u,%u %ux%u %u)", meta->roi_type, meta->x, meta->y, meta->w, + meta->h, num_faces); + + written_roi_ids[roi_id] = TRUE; + offset += ROI_EXTHDR_SIZE; } - return 0; + return offset; } static gboolean @@ -162,50 +295,63 @@ gst_rtp_header_extension_roi_read (GstRTPHeaderExtension * ext, GstBuffer * buffer) { GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); - guint16 roi_type = GST_READ_UINT8 (&data[8]); - - /* safety mechanism to only consider those roi-ext-hdr types - * which we care about */ - if (roi_type == ROI_EXTHDR_TYPE) { - guint16 x = GST_READ_UINT16_BE (&data[0]); - guint16 y = GST_READ_UINT16_BE (&data[2]); - guint16 w = GST_READ_UINT16_BE (&data[4]); - guint16 h = GST_READ_UINT16_BE (&data[6]); - // guint16 roi_type = GST_READ_UINT8 (&bytes[8]); - guint16 num_faces = GST_READ_UINT16_BE (&data[9]); - - GstVideoRegionOfInterestMeta *meta = - gst_buffer_add_video_region_of_interest_meta_id (buffer, - self->roi_type, x, y, w, h); - - GstStructure *extra_param_s = gst_structure_new ("extra-param", - "num_faces", G_TYPE_UINT, num_faces, NULL); - // FIXME: Issue 23027 - // If these asserts got hit the problem in header corruption - // Need to look at re-transmit (?) - // g_assert (w > 0); - // g_assert (h > 0); - gst_video_region_of_interest_meta_add_param (meta, extra_param_s); + /* read as many extension headers as we have signaled roi-types + * XXX: Is this a valid assumption? it makes this only work with + * GStreamer payloader */ + guint offset = 0; + for (guint i = 0; i < g_hash_table_size (self->roi_types); i++) { + guint16 roi_id = GST_READ_UINT8 (&data[offset + 8]); + /* filter by roi-id (roi-type stored in the header extension) */ + gpointer rtp_type_ptr = + g_hash_table_lookup (self->roi_ids, GUINT_TO_POINTER (roi_id)); + guint roi_type = GPOINTER_TO_UINT (rtp_type_ptr); + if (rtp_type_ptr) { + guint16 x = GST_READ_UINT16_BE (&data[offset + 0]); + guint16 y = GST_READ_UINT16_BE (&data[offset + 2]); + guint16 w = GST_READ_UINT16_BE (&data[offset + 4]); + guint16 h = GST_READ_UINT16_BE (&data[offset + 6]); + guint16 num_faces = GST_READ_UINT16_BE (&data[offset + 9]); + GstVideoRegionOfInterestMeta *meta = + gst_buffer_add_video_region_of_interest_meta_id (buffer, + GPOINTER_TO_UINT (rtp_type_ptr), x, y, w, h); + GstStructure *extra_param_s = gst_structure_new ("extra-param", + "num_faces", G_TYPE_UINT, num_faces, NULL); + gst_video_region_of_interest_meta_add_param (meta, extra_param_s); + + GST_TRACE_OBJECT (self, "Read header extension and added RoI meta " + "(%u %u,%u %ux%u %u) to buffer %p", roi_type, + x, y, w, h, num_faces, buffer); + } + offset += ROI_EXTHDR_SIZE; } - return TRUE; } +static void +gst_rtp_header_extension_roi_dispose (GObject * object) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (object); + g_assert (self->roi_types); + g_assert (self->roi_ids); + g_hash_table_destroy (self->roi_types); + g_hash_table_destroy (self->roi_ids); + G_OBJECT_CLASS (gst_rtp_header_extension_roi_parent_class)->dispose (object); +} + static void gst_rtp_header_extension_roi_class_init (GstRTPHeaderExtensionRoiClass * klass) { GstRTPHeaderExtensionClass *rtp_hdr_class; GstElementClass *gstelement_class; GObjectClass *gobject_class; - rtp_hdr_class = GST_RTP_HEADER_EXTENSION_CLASS (klass); gobject_class = (GObjectClass *) klass; gstelement_class = GST_ELEMENT_CLASS (klass); - + gobject_class->dispose = + GST_DEBUG_FUNCPTR (gst_rtp_header_extension_roi_dispose); gobject_class->get_property = gst_rtp_header_extension_roi_get_property; gobject_class->set_property = gst_rtp_header_extension_roi_set_property; - /** * rtphdrextroi:roi-type: * @@ -213,24 +359,28 @@ gst_rtp_header_extension_roi_class_init (GstRTPHeaderExtensionRoiClass * klass) * * Since: 1.20 */ - g_object_class_install_property (gobject_class, PROP_ROI_TYPE, - g_param_spec_uint ("roi-type", "ROI TYPE", - "What roi-type (GQuark) to write the extension-header for", - 0, G_MAXUINT32, 0, + g_object_class_install_property (gobject_class, PROP_ROI_TYPES, + gst_param_spec_array ("roi-types", "RoI types", + "Set of what roi-type(s) (GQuark(s)) to write the extension-header for", + g_param_spec_uint ("roi-type", "RoI type", + "What roi-type (GQuark) to write the extension-header for", + 1, G_MAXUINT32, DEFAULT_ROI_TYPE, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS), G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); - rtp_hdr_class->get_supported_flags = gst_rtp_header_extension_roi_get_supported_flags; rtp_hdr_class->get_max_size = gst_rtp_header_extension_roi_get_max_size; rtp_hdr_class->write = gst_rtp_header_extension_roi_write; rtp_hdr_class->read = gst_rtp_header_extension_roi_read; - gst_element_class_set_static_metadata (gstelement_class, "Region-of-Interest (ROI) RTP Header Extension", GST_RTP_HDREXT_ELEMENT_CLASS, "Region-of-Interest (ROI) RTP Header Extension", "Havard Graff "); gst_rtp_header_extension_class_set_uri (rtp_hdr_class, ROI_HDR_EXT_URI); + GST_DEBUG_CATEGORY_INIT (rtp_hdrext_roi_debug, "rtphdrextroi", 0, + "RTPHeaderExtensionRoI"); } static void diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c b/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c index f805415a1f2..a975b4f27b7 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c @@ -31,6 +31,8 @@ #define SRC_CAPS_STR "video/x-raw,format=I420" +#define MAX_ROI_TYPES_ALLOWED 15 + GST_START_TEST (test_rtphdrext_roi_basic) { GstHarness *h; @@ -152,9 +154,7 @@ GST_START_TEST (test_rtphdrext_roi_default_roi_type) GstMeta *meta; const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; - const GQuark default_roi_type = 0; - GQuark pay_roi_type = ~default_roi_type; - GQuark depay_roi_type = ~default_roi_type; + const GQuark default_roi_type = 126; gpointer state = NULL; gint num_roi_metas_found = 0; @@ -180,11 +180,6 @@ GST_START_TEST (test_rtphdrext_roi_default_roi_type) g_signal_emit_by_name (pay, "add-extension", pay_ext); g_signal_emit_by_name (depay, "add-extension", depay_ext); - g_object_get (pay_ext, "roi-type", &pay_roi_type, NULL); - g_object_get (depay_ext, "roi-type", &depay_roi_type, NULL); - fail_unless_equals_int (pay_roi_type, default_roi_type); - fail_unless_equals_int (depay_roi_type, default_roi_type); - /* verify that we can push and pull buffers */ buf = _gst_harness_create_raw_buffer (h, src_caps); fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); @@ -338,7 +333,8 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type) GstMeta *meta; const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; - const GQuark other_roi_type = 0xFFFFFFFF; + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + const GQuark other_roi_type = 12345; gpointer state = NULL; gint num_roi_metas_found = 0; @@ -351,12 +347,24 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type) pay = gst_harness_find_element (h, "rtpvrawpay"); depay = gst_harness_find_element (h, "rtpvrawdepay"); + gst_value_array_init (&roi_types, 1); + g_value_init (&roi_type, G_TYPE_UINT); + g_value_set_uint (&roi_type, other_roi_type); + gst_value_array_append_value (&roi_types, &roi_type); + + /* explicitly set a roi-type on both payloader and depayloader extensions + * and expect a buffer with such roi-type added to their RoI meta to be + * correctly payloaded and depayloaded */ pay_ext = GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", NULL)); depay_ext = GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", NULL)); + g_object_set_property ((GObject *) pay_ext, "roi-types", &roi_types); + g_object_set_property ((GObject *) depay_ext, "roi-types", &roi_types); + g_value_unset (&roi_type); + g_value_unset (&roi_types); gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); @@ -380,12 +388,6 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type) gst_caps_unref (pay_pad_caps); gst_caps_unref (expected_caps); - /* explicitly set a roi-type on both payloader and depayloader extensions - * and expect a buffer with such roi-type added to their RoI meta to be - * correctly payloaded and depayloaded */ - g_object_set (pay_ext, "roi-type", other_roi_type, NULL); - g_object_set (depay_ext, "roi-type", other_roi_type, NULL); - buf = _gst_harness_create_raw_buffer (h, src_caps); gst_buffer_add_video_region_of_interest_meta_id (buf, other_roi_type, 0, 0, 640, 360); @@ -432,11 +434,202 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_pay_only) GstBuffer *buf; + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + + const GQuark default_roi_type = 126; + const GQuark other_roi_type = default_roi_type / 2; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + /* set a roi-type on the payloader only and expect this to NOT be + * depayloaded as the depayloader will only depayload data using a specific + * set of specified roi-types */ + gst_value_array_init (&roi_types, 1); + g_value_init (&roi_type, G_TYPE_UINT); + g_value_set_uint (&roi_type, other_roi_type); + gst_value_array_append_value (&roi_types, &roi_type); + + g_object_set_property (G_OBJECT (pay_ext), "roi-types", &roi_types); + + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_depay_only) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + + const GQuark default_roi_type = 126; + const GQuark other_roi_type = default_roi_type / 2; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + /* set a roi-type on the depayloader and expect nothing to be depayloaded + * nor payloaded as even if a buffer has RoI meta we would have not signal + * the rigt roi-type to payload, hence no header extensions would be added */ + gst_value_array_init (&roi_types, 1); + g_value_init (&roi_type, G_TYPE_UINT); + g_value_set_uint (&roi_type, other_roi_type); + gst_value_array_append_value (&roi_types, &roi_type); + + g_object_set_property (G_OBJECT (depay_ext), "roi-types", &roi_types); + + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get no metas as we would not be able to payload an + * unknown type hence no RTP header extension is added and the depayloader + * wont, ofcourse add any RoI metas on the output buffer */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_multiple_metas_with_same_roi_type) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + GstMeta *meta; const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; - const GQuark default_roi_type = 0; - const GQuark other_roi_type = ~default_roi_type; + const GQuark default_roi_type = 126; gpointer state = NULL; gint num_roi_metas_found = 0; @@ -483,16 +676,16 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_pay_only) fail_if (gst_buffer_get_meta (buf, meta_info->api)); gst_buffer_unref (buf); - /* set a roi-type on the payloader only and expect this to be - * partially correctly depayloaded as the depayloader will add a meta for - * ANY HDREXT-ROI that has been payloaded by a GStreamer payloader with - * the added rtphdrext-roi element. However, the roi-type on the RoI meta - * would be the default one for the depayloader */ - g_object_set (pay_ext, "roi-type", other_roi_type, NULL); - + /* add multiple RoI meta on an input buffer all with the same default RoI type + * and expect only the first one to be be payloaded and depayloaded back as + * a RoI meta on the output buffer */ buf = _gst_harness_create_raw_buffer (h, src_caps); gst_buffer_add_video_region_of_interest_meta_id (buf, - other_roi_type, 0, 0, 640, 360); + default_roi_type, 0, 0, 640, 360); + gst_buffer_add_video_region_of_interest_meta_id (buf, + default_roi_type, 1, 1, 640, 360); + gst_buffer_add_video_region_of_interest_meta_id (buf, + default_roi_type, 2, 2, 640, 360); fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); buf = gst_harness_pull (h); @@ -525,7 +718,7 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_pay_only) GST_END_TEST; -GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_depay_only) +GST_START_TEST (test_rtphdrext_roi_multiple_roi_types) { GstHarness *h; GstCaps *src_caps, *pay_pad_caps, *expected_caps; @@ -536,10 +729,15 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_depay_only) GstBuffer *buf; + GstMeta *meta; const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; - const GQuark default_roi_type = 0; - const GQuark other_roi_type = ~default_roi_type; + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + const GQuark roi_type_base = 0xDEADBEEF; + + gpointer state = NULL; + gint num_roi_metas_found = 0; + guint i = 0; h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); @@ -556,6 +754,18 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_depay_only) GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", NULL)); + /* allow all roi-types from 1 to MAX_ROI_TYPES_ALLOWED on both the depay and pay ext */ + gst_value_array_init (&roi_types, MAX_ROI_TYPES_ALLOWED); + g_value_init (&roi_type, G_TYPE_UINT); + for (i = 0; i < MAX_ROI_TYPES_ALLOWED; i++) { + g_value_set_uint (&roi_type, roi_type_base + i); + gst_value_array_append_value (&roi_types, &roi_type); + } + g_object_set_property ((GObject *) pay_ext, "roi-types", &roi_types); + g_object_set_property ((GObject *) depay_ext, "roi-types", &roi_types); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); @@ -583,21 +793,70 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_depay_only) fail_if (gst_buffer_get_meta (buf, meta_info->api)); gst_buffer_unref (buf); - /* set a roi-type on the depayloader only */ - g_object_set (depay_ext, "roi-type", other_roi_type, NULL); + /* add a RoI meta for each one of the allowed roi-types an the input buffer + * and expect all to be payloaded and depayloaded back as a RoI metas on + * the output buffer */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + for (i = 0; i < MAX_ROI_TYPES_ALLOWED; i++) { + /* re-use x,y coordinates to make all RoI coordinates different */ + gst_buffer_add_video_region_of_interest_meta_id (buf, + roi_type_base + i, i, i, 640, 360); + } + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly MAX_ROI_TYPES_ALLOWED RoI metas on the output buffer with + * the expected RoI coordinates and type */ + i = 0; + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int (roi_type_base + i, (gint) vmeta->roi_type); + fail_unless_equals_int (i, vmeta->x); + fail_unless_equals_int (i, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + i++; + } + } + fail_unless_equals_int (MAX_ROI_TYPES_ALLOWED, num_roi_metas_found); + gst_buffer_unref (buf); + /* now simply add half as many RoI metas and expect them all to be paylaoded + * and depayloaded */ + num_roi_metas_found = 0; buf = _gst_harness_create_raw_buffer (h, src_caps); - gst_buffer_add_video_region_of_interest_meta_id (buf, - other_roi_type, 0, 0, 640, 360); + for (i = 0; i < MAX_ROI_TYPES_ALLOWED / 2; i++) { + /* re-use x,y coordinates to make all RoI coordinates different */ + gst_buffer_add_video_region_of_interest_meta_id (buf, + roi_type_base + i, i, i, 640, 360); + } fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); buf = gst_harness_pull (h); fail_unless (buf); - /* verify that we get no metas as we would not be able to payload an - * unknown type hence no RTP header extension is added and the depayloader - * wont, ofcourse add any RoI metas on the output buffer */ - fail_if (gst_buffer_get_meta (buf, meta_info->api)); + /* verify that we get exactly MAX_ROI_TYPES_ALLOWED RoI metas on the output buffer with + * the expected RoI coordinates and type */ + i = 0; + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int (roi_type_base + i, (gint) vmeta->roi_type); + fail_unless_equals_int (i, vmeta->x); + fail_unless_equals_int (i, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + i++; + } + } + fail_unless_equals_int (MAX_ROI_TYPES_ALLOWED / 2, num_roi_metas_found); gst_buffer_unref (buf); gst_object_unref (pay_ext); @@ -610,6 +869,114 @@ GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_depay_only) GST_END_TEST; +GST_START_TEST (test_rtphdrext_roi_property_roi_types) +{ + GstRTPHeaderExtension *ext; + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + GValue invalid_roi_type = G_VALUE_INIT; + const GValue *val_ref; + const GQuark default_roi_type = 126; + gboolean seen_roi_type[23] = { 0 }; + + ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + /* Case 1: default roi-types: + * roi-types must be a GValue array of length 1 with a 0 roi-type */ + gst_value_array_init (&roi_types, 1); + g_object_get_property ((GObject *) ext, "roi-types", &roi_types); + fail_unless_equals_int (1, (gint) gst_value_array_get_size (&roi_types)); + val_ref = gst_value_array_get_value (&roi_types, 0); + fail_unless (G_VALUE_HOLDS_UINT (val_ref)); + fail_unless_equals_int (default_roi_type, g_value_get_uint (val_ref)); + g_value_unset (&roi_types); + + /* Case 2: set a valid GValue array with only uints */ + gst_value_array_init (&roi_types, MAX_ROI_TYPES_ALLOWED); + g_value_init (&roi_type, G_TYPE_UINT); + for (guint i = 1; i <= MAX_ROI_TYPES_ALLOWED; i++) { + g_value_set_uint (&roi_type, i); + gst_value_array_append_value (&roi_types, &roi_type); + } + g_object_set_property ((GObject *) ext, "roi-types", &roi_types); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + g_object_get_property ((GObject *) ext, "roi-types", &roi_types); + fail_unless_equals_int (MAX_ROI_TYPES_ALLOWED, + gst_value_array_get_size (&roi_types)); + for (guint i = 0; i < MAX_ROI_TYPES_ALLOWED; i++) { + const GValue *v = gst_value_array_get_value (&roi_types, i); + fail_unless (G_VALUE_HOLDS_UINT (v)); + fail_if (seen_roi_type[g_value_get_uint (v)]); + seen_roi_type[g_value_get_uint (v)] = TRUE; + } + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + /* Case 3: invalid roi-type within GValue array */ + gst_value_array_init (&roi_types, 1); + g_value_init (&roi_type, G_TYPE_UINT); + /* 0 is forbiden as a roi-type */ + g_value_set_uint (&roi_type, 0); + gst_value_array_append_value (&roi_types, &roi_type); + + // FIXME: ASSERT_WARNING broken + // ASSERT_WARNING (g_object_set_property ((GObject *) ext, "roi-types", + // &roi_types)); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + /* Case 4: invalid roi-type within GValue array */ + gst_value_array_init (&roi_types, 1); + g_value_init (&invalid_roi_type, G_TYPE_STRING); + g_value_set_string (&invalid_roi_type, "Dummy"); + gst_value_array_append_value (&roi_types, &invalid_roi_type); + + // FIXME: ASSERT_WARNING broken + // ASSERT_WARNING (g_object_set_property ((GObject *) ext, "roi-types", + // &roi_types)); + g_value_unset (&invalid_roi_type); + g_value_unset (&roi_types); + + /* Case 5: duplicate roi-types on GValue array */ + gst_value_array_init (&roi_types, 3); + g_value_init (&roi_type, G_TYPE_UINT); + /* valid */ + g_value_set_uint (&roi_type, 1); + gst_value_array_append_value (&roi_types, &roi_type); + /* valid */ + g_value_set_uint (&roi_type, 2); + gst_value_array_append_value (&roi_types, &roi_type); + /* duplicate */ + g_value_set_uint (&roi_type, 2); + gst_value_array_append_value (&roi_types, &roi_type); + + // FIXME: ASSERT_WARNING broken + // ASSERT_WARNING (g_object_set_property ((GObject *) ext, "roi-types", + // &roi_types)); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + /* Case 5: invalid length */ + gst_value_array_init (&roi_types, 500); + g_value_init (&roi_type, G_TYPE_UINT); + for (guint i = 0; i < 500; i++) { + g_value_set_uint (&roi_type, i); + gst_value_array_append_value (&roi_types, &roi_type); + } + + // FIXME: ASSERT_WARNING broken + // ASSERT_WARNING (g_object_set_property ((GObject *) ext, "roi-types", + // &roi_types)); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + gst_object_unref (ext); +} + +GST_END_TEST; static Suite * rtphdrext_roi_suite (void) @@ -626,6 +993,12 @@ rtphdrext_roi_suite (void) tcase_add_test (tc_chain, test_rtphdrext_roi_explicit_roi_type_pay_only); tcase_add_test (tc_chain, test_rtphdrext_roi_explicit_roi_type_depay_only); + tcase_add_test (tc_chain, + test_rtphdrext_roi_multiple_metas_with_same_roi_type); + tcase_add_test (tc_chain, test_rtphdrext_roi_multiple_roi_types); + + tcase_add_test (tc_chain, test_rtphdrext_roi_property_roi_types); + return s; } From 3cff5dc8bab804817a9bbc25c1d0bb8aeba59682 Mon Sep 17 00:00:00 2001 From: Will Miller Date: Tue, 14 Dec 2021 14:08:04 +0000 Subject: [PATCH 226/339] gstelement: log warning when pad name doesn't match any template --- subprojects/gstreamer/gst/gstelement.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/subprojects/gstreamer/gst/gstelement.c b/subprojects/gstreamer/gst/gstelement.c index 08b2b077be4..e5919a21376 100644 --- a/subprojects/gstreamer/gst/gstelement.c +++ b/subprojects/gstreamer/gst/gstelement.c @@ -1269,8 +1269,11 @@ gst_element_request_pad_simple (GstElement * element, const gchar * name) } } - if (!templ_found) + if (!templ_found) { + GST_CAT_WARNING (GST_CAT_PADS, "Could not find template in %" GST_PTR_FORMAT + " to match requested pad name %s", element, name); return NULL; + } pad = _gst_element_request_pad (element, templ, req_name, NULL); From 7919314025fe99561b75ddc7236e67ce1ce09b45 Mon Sep 17 00:00:00 2001 From: "sergei.kovalev" Date: Mon, 20 Dec 2021 17:31:17 +0100 Subject: [PATCH 227/339] check: Fix valgrid suppresion for debug function list Fix suppresion to support release and debug builds. Here is debug build call stack: ``` ==10707== by 0x48B5520: g_malloc (gmem.c:106) ==10707== by 0x48D19DC: g_slice_alloc (gslice.c:1069) ==10707== by 0x48D3947: g_slist_copy_deep (gslist.c:619) ==10707== by 0x48D38B8: g_slist_copy (gslist.c:567) ==10707== by 0x4ADC90B: gst_debug_remove_with_compare_func (gstinfo.c:1504) ``` In release build `g_slist_copy (gslist.c:567)` got inlined: ``` ==15419== by 0x48963E0: g_malloc (gmem.c:106) ==15419== by 0x48AA382: g_slice_alloc (gslice.c:1069) ==15419== by 0x48AB732: g_slist_copy_deep (gslist.c:619) ==15419== by 0x4A39B8F: gst_debug_remove_with_compare_func (gstinfo.c:1504) ``` --- subprojects/gstreamer/tests/check/gstreamer.supp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gstreamer/tests/check/gstreamer.supp b/subprojects/gstreamer/tests/check/gstreamer.supp index 3729dfd2b43..586efeadcd4 100644 --- a/subprojects/gstreamer/tests/check/gstreamer.supp +++ b/subprojects/gstreamer/tests/check/gstreamer.supp @@ -4040,7 +4040,7 @@ Memcheck:Leak fun:*alloc ... - fun:g_slist_copy_deep + fun:g_slist_copy* fun:gst_debug_remove_with_compare_func } From 44f55944aee977b275893c681ed3b87dbfbee74a Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Tue, 1 Mar 2022 15:28:21 +0900 Subject: [PATCH 228/339] gst/gst_gdb: allow doing a gst-dot of a local bin --- .../gstreamer/libs/gst/helpers/gst_gdb.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py b/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py index f8273e27964..1bf4674524e 100644 --- a/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py +++ b/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py @@ -1096,24 +1096,31 @@ class GstDot(gdb.Command): """\ Create a pipeline dot file as close as possible to the output of GST_DEBUG_BIN_TO_DOT_FILE. This command will find the top-level parent -for the given gstreamer object and create the dot for that element. +for the given gstreamer object and create the dot representation +for that element or allow the creation of a 'local' dot representation +iff the given object is a valid bin and the optional argument 'local' +is provided in the argument list. -Usage: gst-dot """ +Usage: gst-dot (local)""" def __init__(self): super(GstDot, self).__init__("gst-dot", gdb.COMMAND_DATA) def invoke(self, arg, from_tty): self.dont_repeat() args = gdb.string_to_argv(arg) - if len(args) != 2: - raise Exception("Usage: gst-dot ") + if ((3 > len(args) < 2) + or (len(args) == 3 and args[2] != "local")): + raise Exception("Usage: gst-dot (local)") value = gdb.parse_and_eval(args[0]) if not value: raise Exception("'%s' is not a valid object" % args[0]) value = gst_object_from_value(value) - value = gst_object_pipeline(value) + if len(args) == 2: + value = gst_object_pipeline(value) + elif not gst_is_bin(value): + raise Exception("'%s' is not a valid bin" % value) dot = GdbGstElement(value).pipeline_dot() file = open(args[1], "w") From b7288295656dc3bf140b53944d683486fb220882 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 14 Mar 2022 10:10:11 -0300 Subject: [PATCH 229/339] gstvideodecoder: set all-headers to TRUE in the GstForceKeyUnit when doing a sync point request all_headers set to FALSE stops the FIR request in gstrtpsession. --- .../gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c index eb5d2f6473a..777de8cc1e3 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c @@ -5288,7 +5288,7 @@ gst_video_decoder_request_sync_point_internal (GstVideoDecoder * dec, "Requesting a new key-unit for frame with deadline %" GST_TIME_FORMAT, GST_TIME_ARGS (deadline)); fku = - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, FALSE, + gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, TRUE, 0); priv->last_force_key_unit_time = deadline; } else { From 779c3cc78f80c1bce698b3f11b1a13a946eef50d Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 8 Apr 2022 16:04:09 -0300 Subject: [PATCH 230/339] gstgdiscreencapsrc: fixed type conversion --- .../gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c index e1771eaead6..beadb6a4a5f 100644 --- a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c +++ b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c @@ -378,7 +378,7 @@ gst_gdiscreencapsrc_get_caps (GstBaseSrc * bsrc, GstCaps * filter) GstCaps *caps; if (src->window > 0) - src->screen_rect = rect_dst = gst_win32_get_window_rect (src->window); + src->screen_rect = rect_dst = gst_win32_get_window_rect ((HWND)src->window); else src->screen_rect = rect_dst = gst_win32_get_monitor_rect (src->monitor); From cfebf20f50cf6464e9d525eb4ade33fe854a96f0 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Mon, 18 Apr 2022 19:01:57 +0900 Subject: [PATCH 231/339] gstglcolorconver: only set target target texture if any Prevents logical error as if defined as a stack variable it could contain garbage data if uninitialized. --- .../gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c index 07c6386954c..ffc059e243e 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c @@ -1432,7 +1432,7 @@ gst_gl_color_convert_fixate_format_target (GstCaps * caps, GstCaps * result) const GstVideoFormatInfo *in_info, *out_info = NULL; const GValue *targets; guint targets_mask = 0; - GstGLTextureTarget target; + GstGLTextureTarget target = GST_GL_TEXTURE_TARGET_NONE; gint min_loss = G_MAXINT; guint i, capslen; @@ -1496,7 +1496,7 @@ gst_gl_color_convert_fixate_format_target (GstCaps * caps, GstCaps * result) if (out_info) gst_structure_set (outs, "format", G_TYPE_STRING, GST_VIDEO_FORMAT_INFO_NAME (out_info), NULL); - if (target) + if (target != GST_GL_TEXTURE_TARGET_NONE) gst_structure_set (outs, "texture-target", G_TYPE_STRING, gst_gl_texture_target_to_string (target), NULL); } From 51beb4e587343852c3e1cebfa46c8065efbd62a5 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Mon, 18 Apr 2022 20:01:44 +0900 Subject: [PATCH 232/339] gstglcolorconvert: fix possible out of bounds read --- .../gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c index ffc059e243e..3d2206bde9a 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c @@ -2713,7 +2713,7 @@ _init_convert (GstGLColorConvert * convert) info->cms_coeff3); } - for (i = info->in_n_textures; i >= 0; i--) { + for (i = info->in_n_textures - 1; i >= 0; i--) { if (info->shader_tex_names[i]) gst_gl_shader_set_uniform_1i (convert->shader, info->shader_tex_names[i], i); From d5b29f26f8f49ece0ba3d3bb728b6906cf482566 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 8 Apr 2022 16:05:50 -0300 Subject: [PATCH 233/339] meson: [msvc] only add warnings 4013, 4101 and 4189 if glib_checks is disabled If the checks are not built, it will leave behind some "unused" variables, so this warnings will always trigger. --- subprojects/gst-plugins-bad/meson.build | 11 +++++++++-- subprojects/gst-plugins-base/meson.build | 9 ++++++++- subprojects/gstreamer/meson.build | 9 ++++++++- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/meson.build b/subprojects/gst-plugins-bad/meson.build index 9852206ec49..5575207c734 100644 --- a/subprojects/gst-plugins-bad/meson.build +++ b/subprojects/gst-plugins-bad/meson.build @@ -73,7 +73,6 @@ if cc.get_id() == 'msvc' msvc_args += cc.get_supported_arguments([ '/we4002', # too many actual parameters for macro 'identifier' '/we4003', # not enough actual parameters for macro 'identifier' - '/we4013', # 'function' undefined; assuming extern returning int '/we4020', # 'function' : too many actual parameters '/we4027', # function declared without formal parameter list '/we4029', # declared formal parameter list different from definition @@ -83,9 +82,17 @@ if cc.get_id() == 'msvc' '/we4053', # one void operand for '?:' '/we4062', # enumerator 'identifier' in switch of enum 'enumeration' is not handled '/we4098', # 'function' : void function returning a value + ]) + + # add these warnings only if checks are enabled, + # if the checks are not built, it will leave behind some "unused" variables + if get_option('glib-checks').enabled() + msvc_args += cc.get_supported_arguments([ + '/we4013', # 'function' undefined; assuming extern returning int '/we4101', # 'identifier' : unreferenced local variable '/we4189', # 'identifier' : local variable is initialized but not referenced - ]) + ]) + endif endif add_project_arguments(msvc_args, language: ['c', 'cpp']) # Disable SAFESEH with MSVC for plugins and libs that use external deps that diff --git a/subprojects/gst-plugins-base/meson.build b/subprojects/gst-plugins-base/meson.build index 20325085fa8..28e38262d29 100644 --- a/subprojects/gst-plugins-base/meson.build +++ b/subprojects/gst-plugins-base/meson.build @@ -79,9 +79,16 @@ if cc.get_id() == 'msvc' '/we4053', # one void operand for '?:' '/we4062', # enumerator 'identifier' in switch of enum 'enumeration' is not handled '/we4098', # 'function' : void function returning a value + ]) + + # add these warnings only if checks are enabled, + # if the checks are not built, it will leave behind some "unused" variables + if get_option('glib-checks').enabled() + msvc_args += cc.get_supported_arguments([ '/we4101', # 'identifier' : unreferenced local variable '/we4189', # 'identifier' : local variable is initialized but not referenced - ]) + ]) + endif endif add_project_arguments(msvc_args, language: ['c', 'cpp']) # Disable SAFESEH with MSVC for plugins and libs that use external deps that diff --git a/subprojects/gstreamer/meson.build b/subprojects/gstreamer/meson.build index a48488fba36..bca7ab86aa0 100644 --- a/subprojects/gstreamer/meson.build +++ b/subprojects/gstreamer/meson.build @@ -76,9 +76,16 @@ if cc.get_id() == 'msvc' '/we4053', # one void operand for '?:' '/we4062', # enumerator 'identifier' in switch of enum 'enumeration' is not handled '/we4098', # 'function' : void function returning a value + ]) + + # add these warnings only if checks are enabled, + # if the checks are not built, it will leave behind some "unused" variables + if get_option('glib-checks').enabled() + msvc_args += cc.get_supported_arguments([ '/we4101', # 'identifier' : unreferenced local variable '/we4189', # 'identifier' : local variable is initialized but not referenced - ]) + ]) + endif endif add_project_arguments(msvc_args, language: 'c') elif cc.has_link_argument('-Wl,-Bsymbolic-functions') From 6750618208423207a4959449580e2aa6b3e03ea1 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Thu, 26 May 2022 15:31:30 +0900 Subject: [PATCH 234/339] rtprtx: add test to copy custom RoI ext-hdr --- .../tests/check/elements/rtprtx.c | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c index f245e3573d1..d3b77d64153 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c @@ -1133,6 +1133,149 @@ GST_START_TEST (test_rtxsender_copy_twcc_exthdr) GST_END_TEST; +static GstBuffer * +create_rtp_buffer_with_roi_ext (guint32 ssrc, guint8 payload_type, + guint16 seqnum, guint payload_size, guint8 ext_id, + guint16 x, guint16 y, guint16 w, guint16 h, guint8 id, guint16 num_faces) +{ + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstBuffer *buf = + create_rtp_buffer_with_payload_size (ssrc, payload_type, seqnum, + payload_size); + + gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp); + + if (ext_id > 0) { + guint8 data[11]; + GST_WRITE_UINT16_BE (&data[0], x); + GST_WRITE_UINT16_BE (&data[2], y); + GST_WRITE_UINT16_BE (&data[4], w); + GST_WRITE_UINT16_BE (&data[6], h); + GST_WRITE_UINT16_BE (&data[9], num_faces); + GST_WRITE_UINT8 (&data[8], id); + gst_rtp_buffer_add_extension_twobytes_header (&rtp, 0, ext_id, + data, sizeof (data)); + } + + gst_rtp_buffer_unmap (&rtp); + + return buf; +} + +static void +read_roi (GstBuffer * buf, guint8 ext_id, + guint16 * x, guint16 * y, guint16 * w, guint16 * h, + guint8 * id, guint16 * num_faces) +{ + gpointer ext_data; + guint ext_size; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + if (!gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp)) + return; + + if (!gst_rtp_buffer_get_extension_twobytes_header (&rtp, NULL, ext_id, 0, + &ext_data, &ext_size)) { + gst_rtp_buffer_unmap (&rtp); + return; + } + + fail_unless (ext_data != NULL); + fail_unless (ext_size == 11); + + { + const guint8 *bytes = (const guint8 *) ext_data; + *x = GST_READ_UINT16_BE (&bytes[0]); + *y = GST_READ_UINT16_BE (&bytes[2]); + *w = GST_READ_UINT16_BE (&bytes[4]); + *h = GST_READ_UINT16_BE (&bytes[6]); + *id = GST_READ_UINT8 (&bytes[8]); + *num_faces = GST_READ_UINT16_BE (&bytes[9]); + } + gst_rtp_buffer_unmap (&rtp); +} + +GST_START_TEST (test_rtxsender_copy_roi_exthdr) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstBuffer *inbuf, *outbuf, *rtxbuf; + guint8 ext_id = 5; + GstHarness *hsend = gst_harness_new ("rtprtxsend"); + GstHarness *hrecv = gst_harness_new ("rtprtxreceive"); + guint16 expected_x = 2, actual_x = 0; + guint16 expected_y = 3, actual_y = 0; + guint16 expected_w = 5, actual_w = 0; + guint16 expected_h = 7, actual_h = 0; + guint8 expected_id = 11, actual_id = 0; + guint16 expected_num_faces = 13, actual_num_faces = 0; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (hsend->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, NULL); + g_object_set (hrecv->element, + "payload-type-map", pt_map, "ssrc-map", ssrc_map, NULL); + + gst_harness_set_src_caps_str (hsend, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + gst_harness_set_src_caps_str (hrecv, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (hsend, 0 * GST_MSECOND); + gst_harness_set_time (hrecv, 0 * GST_MSECOND); + + /* create buffer with RoI extension on it */ + inbuf = create_rtp_buffer_with_roi_ext (master_ssrc, master_pt, 0, 20, + ext_id, expected_x, expected_y, expected_w, expected_h, + expected_id, expected_num_faces); + + /* push packets through rtxsend to rtxreceive */ + gst_harness_push (hsend, gst_buffer_ref (inbuf)); + gst_harness_push (hrecv, gst_harness_pull (hsend)); + pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 0); + + /* get rid of reconfigure event in preparation for next step */ + gst_event_unref (gst_harness_pull_upstream_event (hrecv)); + fail_unless_equals_int (gst_harness_upstream_events_in_queue (hrecv), 0); + + /* push RTX events through rtxsend to rtxreceive */ + gst_harness_push_upstream_event (hrecv, + create_rtx_event (master_ssrc, master_pt, 0)); + gst_harness_push_upstream_event (hsend, + gst_harness_pull_upstream_event (hrecv)); + + rtxbuf = gst_harness_pull (hsend); + verify_buf (rtxbuf, TRUE, rtx_ssrc, rtx_pt, 0); + read_roi (rtxbuf, ext_id, &actual_x, &actual_y, &actual_w, &actual_h, + &actual_id, &actual_num_faces); + fail_unless_equals_int (expected_x, actual_x); + fail_unless_equals_int (expected_y, actual_y); + fail_unless_equals_int (expected_w, actual_w); + fail_unless_equals_int (expected_h, actual_h); + fail_unless_equals_int (expected_id, actual_id); + fail_unless_equals_int (expected_num_faces, actual_num_faces); + gst_harness_push (hrecv, rtxbuf); + + outbuf = gst_harness_pull (hrecv); + compare_rtp_packets (inbuf, outbuf); + gst_buffer_unref (inbuf); + gst_buffer_unref (outbuf); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (hsend); + gst_harness_teardown (hrecv); +} + +GST_END_TEST; + #define RTPHDREXT_STREAM_ID GST_RTP_HDREXT_BASE "sdes:rtp-stream-id" #define RTPHDREXT_REPAIRED_STREAM_ID GST_RTP_HDREXT_BASE "sdes:repaired-rtp-stream-id" @@ -1725,6 +1868,7 @@ rtprtx_suite (void) tcase_add_test (tc_chain, test_rtxsend_header_extensions); tcase_add_test (tc_chain, test_rtxsend_header_extensions_copy); tcase_add_test (tc_chain, test_rtxsender_copy_twcc_exthdr); + tcase_add_test (tc_chain, test_rtxsender_copy_roi_exthdr); tcase_add_test (tc_chain, test_rtxsender_stuffing); tcase_add_test (tc_chain, test_rtxsender_stuffing_toggle); From 78fbedccb30c440c221e7537f9035e8ef305d083 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 6 Apr 2022 12:13:46 -0300 Subject: [PATCH 235/339] rtprtx: fix copying ext-hdr if present --- .../gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c index 5d74c235031..b64c12b0bac 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c @@ -728,7 +728,12 @@ _gst_rtp_buffer_new_from_rtx (GstRtpRtxReceive * rtx, GstRTPBuffer * rtp, /* copy extension if any */ if (rtp->size[1]) { - mem = rewrite_header_extensions (rtx, rtp); + if (rtx->rid_stream) { + mem = rewrite_header_extensions (rtx, rtp); + } else { + mem = gst_memory_copy (rtp->map[1].memory, + (guint8 *) rtp->data[1] - rtp->map[1].data, rtp->size[1]); + } gst_buffer_append_memory (new_buffer, mem); } From 937d853954c84363ef68943fd2f36d03ccd60019 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Sat, 11 Jun 2022 16:19:57 +0900 Subject: [PATCH 236/339] libcheck: use SIGABRT instead of SIGKILL on timeout This allows user-level signal handler to catch this Co-authored-by: Frederik Vestre --- subprojects/gstreamer/libs/gst/check/libcheck/check_run.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c b/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c index 82bd83ea88d..0ce9c6473f3 100644 --- a/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c +++ b/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c @@ -106,7 +106,7 @@ sig_handler (int sig_nr) switch (sig_nr) { case SIGALRM: alarm_received = 1; - killpg (group_pid, SIGKILL); + killpg (group_pid, SIGABRT); break; case SIGTERM: case SIGINT: From 5ee60edcfb92f417c798907711cb70ea89e3f9bc Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 25 Apr 2022 14:16:39 +0200 Subject: [PATCH 237/339] ahc2src: Introduce a new source for Android Camera 2 NDK API --- .../sys/androidmedia/gstahc2src.c | 2111 +++++++++++++++++ .../sys/androidmedia/gstahc2src.h | 56 + .../gst-plugins-bad/sys/androidmedia/gstamc.c | 9 + .../sys/androidmedia/meson.build | 11 +- 4 files changed, 2186 insertions(+), 1 deletion(-) create mode 100644 subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c create mode 100644 subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.h diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c new file mode 100644 index 00000000000..8107f36bb92 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c @@ -0,0 +1,2111 @@ +/* GStreamer android.hardware.Camera2 Source + * Copyright (C) 2017, Collabora Ltd. + * Author:Justin Kim + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "gstahc2src.h" + +#include +#include +#include +#include + +#define GST_USE_UNSTABLE_API +#include + +#include +/* FIXME: NdkCameraManager.h uses 'ACameraDevice_StateCallbacks' + * instead of a small 's'.*/ +#ifndef ACameraDevice_StateCallbacks +#define ACameraDevice_StateCallbacks ACameraDevice_stateCallbacks +#endif +#include + +#include +#include + +#define DEFAULT_MAX_IMAGES 5 + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +GST_DEBUG_CATEGORY_STATIC (gst_debug_ahc2_src); +#define GST_CAT_DEFAULT (gst_debug_ahc2_src) + +struct _GstAHC2Src +{ + GstPushSrc parent; + + GstDataQueue *outbound_queue; + GstClockTime previous_ts; + gboolean started; + GMutex mutex; + gint max_images; + + gint camera_index; + ACameraDevice *camera_device; + ACameraManager *camera_manager; + ACameraIdList *camera_id_list; + gint camera_template_type; + ACameraOutputTarget *camera_output_target; + ACameraCaptureSession *camera_capture_session; + + AImageReader *image_reader; + AImageReader_ImageListener image_reader_listener; + + ACaptureRequest *capture_request; + ACaptureSessionOutput *capture_sout; + ACaptureSessionOutputContainer *capture_sout_container; + + ACameraDevice_stateCallbacks device_state_cb; + ACameraCaptureSession_stateCallbacks session_state_cb; +}; + +typedef struct +{ + gsize refcount; + GstAHC2Src *ahc2src; + AImage *image; +} GstWrappedAImage; + +#define GST_TYPE_WRAPPED_AIMAGE (gst_wrapped_aimage_get_type ()) +GType gst_wrapped_aimage_get_type (void); + +static GstWrappedAImage * +gst_wrapped_aimage_ref (GstWrappedAImage * self) +{ + g_return_val_if_fail (self != NULL, NULL); + g_return_val_if_fail (self->ahc2src != NULL, NULL); + g_return_val_if_fail (self->image != NULL, NULL); + g_return_val_if_fail (self->refcount >= 1, NULL); + + GST_TRACE_OBJECT (self->ahc2src, + "ref GstWrappedAImage %p, refcount: %" G_GSIZE_FORMAT, self, + self->refcount); + + self->refcount++; + return self; +} + +static void +gst_wrapped_aimage_unref (GstWrappedAImage * self) +{ + g_return_if_fail (self != NULL); + g_return_if_fail (self->ahc2src != NULL); + g_return_if_fail (self->image != NULL); + g_return_if_fail (self->refcount >= 1); + + GST_TRACE_OBJECT (self->ahc2src, + "unref GstWrappedAImage %p, refcount: %" G_GSIZE_FORMAT, self, + self->refcount); + if (--self->refcount == 0) { + gst_object_unref (self->ahc2src); + g_clear_pointer (&self->image, (GDestroyNotify) AImage_delete); + } +} + +G_DEFINE_BOXED_TYPE (GstWrappedAImage, gst_wrapped_aimage, + gst_wrapped_aimage_ref, gst_wrapped_aimage_unref); + +static void gst_ahc2_src_photography_init (gpointer g_iface, + gpointer iface_data); + +#define gst_ahc2_src_parent_class parent_class + +G_DEFINE_TYPE_WITH_CODE (GstAHC2Src, gst_ahc2_src, GST_TYPE_PUSH_SRC, + G_IMPLEMENT_INTERFACE (GST_TYPE_PHOTOGRAPHY, + gst_ahc2_src_photography_init)); + +typedef enum +{ + PROP_CAMERA_INDEX = 1, + PROP_N_CAMERAS, + PROP_CAMERA_TEMPLATE_TYPE, + PROP_MAX_IMAGES, + + /* override photography properties */ + PROP_ANALOG_GAIN, + PROP_APERTURE, + PROP_CAPABILITIES, + PROP_COLOR_TEMPERATURE, + PROP_COLOR_TONE_MODE, + PROP_EV_COMPENSATION, + PROP_EXPOSURE_TIME, + PROP_FLASH_MODE, + PROP_FLICKER_MODE, + PROP_FOCUS_MODE, + PROP_IMAGE_CAPTURE_SUPPORTED_CAPS, + PROP_IMAGE_PREVIEW_SUPPORTED_CAPS, + PROP_ISO_SPEED, + PROP_LENS_FOCUS, + PROP_MAX_EXPOSURE_TIME, + PROP_MIN_EXPOSURE_TIME, + PROP_NOISE_REDUCTION, + PROP_SCENE_MODE, + PROP_WHITE_BALANCE_MODE, + PROP_WHITE_POINT, + PROP_ZOOM, + + /*< private > */ + PROP_N_INSTALL = PROP_MAX_IMAGES + 1, + PROP_LAST = PROP_ZOOM, +} GstAHC2SrcProperty; + +static GParamSpec *properties[PROP_LAST + 1]; + +enum +{ + SIG_GET_CAMERA_ID_BY_INDEX, + LAST_SIGNAL +}; + +static guint signals[LAST_SIGNAL] = { 0 }; + +static GstPhotographyCaps +gst_ahc2_src_get_capabilities (GstPhotography * photo) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + GstPhotographyCaps caps = GST_PHOTOGRAPHY_CAPS_EV_COMP | + GST_PHOTOGRAPHY_CAPS_WB_MODE | GST_PHOTOGRAPHY_CAPS_TONE | + GST_PHOTOGRAPHY_CAPS_SCENE | GST_PHOTOGRAPHY_CAPS_FLASH | + GST_PHOTOGRAPHY_CAPS_FOCUS; + + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->camera_manager != NULL, caps); + g_return_val_if_fail (self->camera_id_list != NULL, caps); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + caps); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + return caps; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_LENS_INFO_AVAILABLE_FOCAL_LENGTHS, &entry) == ACAMERA_OK) { + if (entry.count > 1) { + caps |= GST_PHOTOGRAPHY_CAPS_ZOOM; + } + } + + g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + + return caps; +} + +static gboolean +gst_ahc2_src_get_color_tone_mode (GstPhotography * photo, + GstPhotographyColorToneMode * tone_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_EFFECT_MODE, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EFFECT MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_EFFECT_MODE_OFF: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_NORMAL; + break; + case ACAMERA_CONTROL_EFFECT_MODE_MONO: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_GRAYSCALE; + break; + case ACAMERA_CONTROL_EFFECT_MODE_NEGATIVE: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_NEGATIVE; + break; + case ACAMERA_CONTROL_EFFECT_MODE_SOLARIZE: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_SOLARIZE; + break; + case ACAMERA_CONTROL_EFFECT_MODE_SEPIA: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_SEPIA; + break; + case ACAMERA_CONTROL_EFFECT_MODE_POSTERIZE: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_POSTERIZE; + break; + case ACAMERA_CONTROL_EFFECT_MODE_WHITEBOARD: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_WHITEBOARD; + break; + case ACAMERA_CONTROL_EFFECT_MODE_BLACKBOARD: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_BLACKBOARD; + break; + case ACAMERA_CONTROL_EFFECT_MODE_AQUA: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_AQUA; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_color_tone_mode (GstPhotography * photo, + GstPhotographyColorToneMode tone_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (tone_mode) { + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_NORMAL: + mode = ACAMERA_CONTROL_EFFECT_MODE_OFF; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_SEPIA: + mode = ACAMERA_CONTROL_EFFECT_MODE_SEPIA; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_NEGATIVE: + mode = ACAMERA_CONTROL_EFFECT_MODE_NEGATIVE; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_GRAYSCALE: + mode = ACAMERA_CONTROL_EFFECT_MODE_MONO; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_SOLARIZE: + mode = ACAMERA_CONTROL_EFFECT_MODE_SOLARIZE; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_POSTERIZE: + mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_POSTERIZE; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_WHITEBOARD: + mode = ACAMERA_CONTROL_EFFECT_MODE_WHITEBOARD; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_BLACKBOARD: + mode = ACAMERA_CONTROL_EFFECT_MODE_BLACKBOARD; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_AQUA: + mode = ACAMERA_CONTROL_EFFECT_MODE_AQUA; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_NATURAL: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_VIVID: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_COLORSWAP: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_OUT_OF_FOCUS: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_SKY_BLUE: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_GRASS_GREEN: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_SKIN_WHITEN: + default: + GST_WARNING_OBJECT (self, "Unsupported color tone (%d)", tone_mode); + return FALSE; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_EFFECT_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static void +gst_ahc2_src_set_autofocus (GstPhotography * photo, gboolean on) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + guint8 mode = on ? ACAMERA_CONTROL_AF_MODE_AUTO : ACAMERA_CONTROL_AF_MODE_OFF; + + g_return_if_fail (self->capture_request != NULL); + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_AF_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); +} + +static gboolean +gst_ahc2_src_get_focus_mode (GstPhotography * photo, + GstPhotographyFocusMode * focus_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_AF_MODE, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get AF MODE"); + return FALSE; + } + + /* GstPhotographyFocusMode doesn't matches + * acamera_metadata_enum_acamera_control_af_mode. */ + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_AF_MODE_OFF: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_MANUAL; + break; + case ACAMERA_CONTROL_AF_MODE_AUTO: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_AUTO; + break; + case ACAMERA_CONTROL_AF_MODE_MACRO: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_MACRO; + break; + case ACAMERA_CONTROL_AF_MODE_CONTINUOUS_VIDEO: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_HYPERFOCAL; + break; + case ACAMERA_CONTROL_AF_MODE_CONTINUOUS_PICTURE: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_CONTINUOUS_NORMAL; + break; + case ACAMERA_CONTROL_AF_MODE_EDOF: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_EXTENDED; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_focus_mode (GstPhotography * photo, + GstPhotographyFocusMode focus_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + guint8 mode; + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (focus_mode) { + case GST_PHOTOGRAPHY_FOCUS_MODE_AUTO: + mode = ACAMERA_CONTROL_AF_MODE_AUTO; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_MACRO: + mode = ACAMERA_CONTROL_AF_MODE_MACRO; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_PORTRAIT: + mode = ACAMERA_CONTROL_AF_MODE_CONTINUOUS_PICTURE; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_INFINITY: + case GST_PHOTOGRAPHY_FOCUS_MODE_HYPERFOCAL: + case GST_PHOTOGRAPHY_FOCUS_MODE_CONTINUOUS_NORMAL: + case GST_PHOTOGRAPHY_FOCUS_MODE_CONTINUOUS_EXTENDED: + mode = ACAMERA_CONTROL_AF_MODE_CONTINUOUS_VIDEO; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_MANUAL: + mode = ACAMERA_CONTROL_AF_MODE_OFF; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_EXTENDED: + mode = ACAMERA_CONTROL_AF_MODE_EDOF; + break; + default: + g_assert_not_reached (); + break; + } + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_AF_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_ev_compensation (GstPhotography * photo, gfloat * ev_comp) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + gint ev; + ACameraMetadata_rational ev_step; + gboolean ret = FALSE; + + g_return_val_if_fail (self->camera_manager != NULL, FALSE); + g_return_val_if_fail (self->camera_id_list != NULL, FALSE); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get metadata object"); + goto out; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_CONTROL_AE_COMPENSATION_STEP, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EV STEP"); + goto out; + } + + ev_step = entry.data.r[0]; + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_AE_EXPOSURE_COMPENSATION, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EV COMPENSATION"); + goto out; + } + + ev = entry.data.i32[0]; + + *ev_comp = (gfloat) (ev * ev_step.numerator / ev_step.denominator); + + ret = TRUE; +out: + g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + return ret; +} + +static gboolean +gst_ahc2_src_set_ev_compensation (GstPhotography * photo, gfloat ev_comp) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + gint ev, ev_min, ev_max; + ACameraMetadata_rational ev_step; + gboolean ret = FALSE; + + g_return_val_if_fail (self->camera_manager != NULL, FALSE); + g_return_val_if_fail (self->camera_id_list != NULL, FALSE); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get metadata object"); + goto out; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_CONTROL_AE_COMPENSATION_STEP, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EV STEP"); + goto out; + } + + ev_step = entry.data.r[0]; + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_CONTROL_AE_COMPENSATION_RANGE, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EV RANGE"); + goto out; + } + + ev_min = entry.data.i32[0]; + ev_max = entry.data.i32[1]; + + ev = (gint) round (ev_comp / ev_step.numerator * ev_step.denominator); + + if (ev < ev_min) { + GST_WARNING_OBJECT (self, "converted EV(%d) is less than min(%d)", ev, + ev_min); + ev = ev_min; + } + + if (ev > ev_max) { + GST_WARNING_OBJECT (self, "converted EV(%d) is greater than max(%d)", ev, + ev_max); + ev = ev_max; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_i32 (self->capture_request, + ACAMERA_CONTROL_AE_EXPOSURE_COMPENSATION, 1, &ev); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + ret = TRUE; +out: + g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + return ret; +} + +static gboolean +gst_ahc2_src_get_flash_mode (GstPhotography * photo, + GstPhotographyFlashMode * flash_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_FLASH_MODE, &entry) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get FLASH MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_FLASH_MODE_OFF: + *flash_mode = GST_PHOTOGRAPHY_FLASH_MODE_OFF; + break; + case ACAMERA_FLASH_MODE_SINGLE: + *flash_mode = GST_PHOTOGRAPHY_FLASH_MODE_AUTO; + break; + case ACAMERA_FLASH_MODE_TORCH: + *flash_mode = GST_PHOTOGRAPHY_FLASH_MODE_ON; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_flash_mode (GstPhotography * photo, + GstPhotographyFlashMode flash_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (flash_mode) { + case GST_PHOTOGRAPHY_FLASH_MODE_OFF: + mode = ACAMERA_FLASH_MODE_OFF; + break; + case GST_PHOTOGRAPHY_FLASH_MODE_ON: + mode = ACAMERA_FLASH_MODE_TORCH; + break; + case GST_PHOTOGRAPHY_FLASH_MODE_AUTO: + case GST_PHOTOGRAPHY_FLASH_MODE_FILL_IN: + case GST_PHOTOGRAPHY_FLASH_MODE_RED_EYE: + mode = ACAMERA_FLASH_MODE_SINGLE; + break; + default: + g_assert_not_reached (); + break; + } + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_FLASH_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_flicker_mode (GstPhotography * photo, + GstPhotographyFlickerReductionMode * flicker_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_AE_ANTIBANDING_MODE, &entry) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get ANTIBANDING MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_AE_ANTIBANDING_MODE_OFF: + *flicker_mode = GST_PHOTOGRAPHY_FLICKER_REDUCTION_OFF; + break; + case ACAMERA_CONTROL_AE_ANTIBANDING_MODE_50HZ: + *flicker_mode = GST_PHOTOGRAPHY_FLICKER_REDUCTION_50HZ; + break; + case ACAMERA_CONTROL_AE_ANTIBANDING_MODE_60HZ: + *flicker_mode = GST_PHOTOGRAPHY_FLICKER_REDUCTION_60HZ; + break; + case ACAMERA_CONTROL_AE_ANTIBANDING_MODE_AUTO: + *flicker_mode = GST_PHOTOGRAPHY_FLICKER_REDUCTION_AUTO; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_flicker_mode (GstPhotography * photo, + GstPhotographyFlickerReductionMode flicker_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (flicker_mode) { + case GST_PHOTOGRAPHY_FLICKER_REDUCTION_OFF: + mode = ACAMERA_CONTROL_AE_ANTIBANDING_MODE_OFF; + break; + case GST_PHOTOGRAPHY_FLICKER_REDUCTION_50HZ: + mode = ACAMERA_CONTROL_AE_ANTIBANDING_MODE_50HZ; + break; + case GST_PHOTOGRAPHY_FLICKER_REDUCTION_60HZ: + mode = ACAMERA_CONTROL_AE_ANTIBANDING_MODE_60HZ; + break; + case GST_PHOTOGRAPHY_FLICKER_REDUCTION_AUTO: + mode = ACAMERA_CONTROL_AE_ANTIBANDING_MODE_AUTO; + break; + default: + GST_WARNING_OBJECT (self, "Unsupported flicker-mode (%d)", flicker_mode); + break; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_AE_ANTIBANDING_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_scene_mode (GstPhotography * photo, + GstPhotographySceneMode * scene_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_SCENE_MODE, &entry) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get SCENE MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_SCENE_MODE_DISABLED: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_MANUAL; + break; + case ACAMERA_CONTROL_SCENE_MODE_ACTION: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_ACTION; + break; + case ACAMERA_CONTROL_SCENE_MODE_PORTRAIT: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_PORTRAIT; + break; + case ACAMERA_CONTROL_SCENE_MODE_LANDSCAPE: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_LANDSCAPE; + break; + case ACAMERA_CONTROL_SCENE_MODE_NIGHT: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_NIGHT; + break; + case ACAMERA_CONTROL_SCENE_MODE_NIGHT_PORTRAIT: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_NIGHT_PORTRAIT; + break; + case ACAMERA_CONTROL_SCENE_MODE_THEATRE: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_THEATRE; + break; + case ACAMERA_CONTROL_SCENE_MODE_BEACH: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_BEACH; + break; + case ACAMERA_CONTROL_SCENE_MODE_SNOW: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_SNOW; + break; + case ACAMERA_CONTROL_SCENE_MODE_SUNSET: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_SUNSET; + break; + case ACAMERA_CONTROL_SCENE_MODE_STEADYPHOTO: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_STEADY_PHOTO; + break; + case ACAMERA_CONTROL_SCENE_MODE_FIREWORKS: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_FIREWORKS; + break; + case ACAMERA_CONTROL_SCENE_MODE_SPORTS: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_SPORT; + break; + case ACAMERA_CONTROL_SCENE_MODE_PARTY: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_PARTY; + break; + case ACAMERA_CONTROL_SCENE_MODE_CANDLELIGHT: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_CANDLELIGHT; + break; + case ACAMERA_CONTROL_SCENE_MODE_BARCODE: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_BARCODE; + break; + case ACAMERA_CONTROL_SCENE_MODE_HDR: + case ACAMERA_CONTROL_SCENE_MODE_FACE_PRIORITY: + GST_WARNING_OBJECT (self, + "Device returns scene mode (%d), but no proper interface is defined", + entry.data.u8[0]); + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_MANUAL; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_scene_mode (GstPhotography * photo, + GstPhotographySceneMode scene_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (scene_mode) { + case GST_PHOTOGRAPHY_SCENE_MODE_MANUAL: + mode = ACAMERA_CONTROL_SCENE_MODE_DISABLED; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_PORTRAIT: + mode = ACAMERA_CONTROL_SCENE_MODE_PORTRAIT; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_LANDSCAPE: + mode = ACAMERA_CONTROL_SCENE_MODE_LANDSCAPE; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_SPORT: + mode = ACAMERA_CONTROL_SCENE_MODE_SPORTS; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_NIGHT: + mode = ACAMERA_CONTROL_SCENE_MODE_NIGHT; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_ACTION: + mode = ACAMERA_CONTROL_SCENE_MODE_ACTION; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_NIGHT_PORTRAIT: + mode = ACAMERA_CONTROL_SCENE_MODE_NIGHT_PORTRAIT; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_THEATRE: + mode = ACAMERA_CONTROL_SCENE_MODE_THEATRE; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_BEACH: + mode = ACAMERA_CONTROL_SCENE_MODE_BEACH; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_SNOW: + mode = ACAMERA_CONTROL_SCENE_MODE_SNOW; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_SUNSET: + mode = ACAMERA_CONTROL_SCENE_MODE_SUNSET; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_STEADY_PHOTO: + mode = ACAMERA_CONTROL_SCENE_MODE_STEADYPHOTO; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_FIREWORKS: + mode = ACAMERA_CONTROL_SCENE_MODE_FIREWORKS; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_PARTY: + mode = ACAMERA_CONTROL_SCENE_MODE_PARTY; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_CANDLELIGHT: + mode = ACAMERA_CONTROL_SCENE_MODE_CANDLELIGHT; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_BARCODE: + mode = ACAMERA_CONTROL_SCENE_MODE_BARCODE; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_AUTO: + case GST_PHOTOGRAPHY_SCENE_MODE_CLOSEUP: + default: + GST_WARNING_OBJECT (self, "Unsupported scene mode (%d)", scene_mode); + return FALSE; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_SCENE_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_white_balance_mode (GstPhotography * photo, + GstPhotographyWhiteBalanceMode * mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_AWB_MODE, &entry) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get AWB_MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_AWB_MODE_OFF: + *mode = GST_PHOTOGRAPHY_WB_MODE_MANUAL; + break; + case ACAMERA_CONTROL_AWB_MODE_AUTO: + *mode = GST_PHOTOGRAPHY_WB_MODE_AUTO; + break; + case ACAMERA_CONTROL_AWB_MODE_INCANDESCENT: + *mode = GST_PHOTOGRAPHY_WB_MODE_TUNGSTEN; + break; + case ACAMERA_CONTROL_AWB_MODE_FLUORESCENT: + *mode = GST_PHOTOGRAPHY_WB_MODE_FLUORESCENT; + break; + case ACAMERA_CONTROL_AWB_MODE_WARM_FLUORESCENT: + *mode = GST_PHOTOGRAPHY_WB_MODE_WARM_FLUORESCENT; + break; + case ACAMERA_CONTROL_AWB_MODE_DAYLIGHT: + *mode = GST_PHOTOGRAPHY_WB_MODE_DAYLIGHT; + break; + case ACAMERA_CONTROL_AWB_MODE_CLOUDY_DAYLIGHT: + *mode = GST_PHOTOGRAPHY_WB_MODE_CLOUDY; + break; + case ACAMERA_CONTROL_AWB_MODE_TWILIGHT: + *mode = GST_PHOTOGRAPHY_WB_MODE_SUNSET; + break; + case ACAMERA_CONTROL_AWB_MODE_SHADE: + *mode = GST_PHOTOGRAPHY_WB_MODE_SHADE; + break; + default: + GST_WARNING_OBJECT (self, "Uknown AWB parameter (0x%x)", + entry.data.u8[0]); + return FALSE; + break; + } + + GST_DEBUG_OBJECT (self, "AWB parameter (0x%x -> 0x%x)", entry.data.u8[0], + *mode); + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_white_balance_mode (GstPhotography * photo, + GstPhotographyWhiteBalanceMode mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 wb_mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (mode) { + case GST_PHOTOGRAPHY_WB_MODE_AUTO: + wb_mode = ACAMERA_CONTROL_AWB_MODE_AUTO; + break; + case GST_PHOTOGRAPHY_WB_MODE_DAYLIGHT: + wb_mode = ACAMERA_CONTROL_AWB_MODE_DAYLIGHT; + break; + case GST_PHOTOGRAPHY_WB_MODE_CLOUDY: + wb_mode = ACAMERA_CONTROL_AWB_MODE_CLOUDY_DAYLIGHT; + break; + case GST_PHOTOGRAPHY_WB_MODE_SUNSET: + wb_mode = ACAMERA_CONTROL_AWB_MODE_TWILIGHT; + break; + case GST_PHOTOGRAPHY_WB_MODE_TUNGSTEN: + wb_mode = ACAMERA_CONTROL_AWB_MODE_INCANDESCENT; + break; + case GST_PHOTOGRAPHY_WB_MODE_FLUORESCENT: + wb_mode = ACAMERA_CONTROL_AWB_MODE_FLUORESCENT; + break; + case GST_PHOTOGRAPHY_WB_MODE_MANUAL: + wb_mode = ACAMERA_CONTROL_AWB_MODE_OFF; + break; + case GST_PHOTOGRAPHY_WB_MODE_WARM_FLUORESCENT: + wb_mode = ACAMERA_CONTROL_AWB_MODE_WARM_FLUORESCENT; + break; + case GST_PHOTOGRAPHY_WB_MODE_SHADE: + wb_mode = ACAMERA_CONTROL_AWB_MODE_SHADE; + break; + default: + g_assert_not_reached (); + break; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_AWB_MODE, 1, &wb_mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_zoom (GstPhotography * photo, gfloat * zoom) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_LENS_FOCAL_LENGTH, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get FOCAL LENGTH (optical zoom)"); + return FALSE; + } + + *zoom = entry.data.f[0]; + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_zoom (GstPhotography * photo, gfloat zoom) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + gboolean ret = FALSE; + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->camera_manager != NULL, FALSE); + g_return_val_if_fail (self->camera_id_list != NULL, FALSE); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get metadata"); + return FALSE; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_LENS_INFO_AVAILABLE_FOCAL_LENGTHS, &entry) != ACAMERA_OK) { + goto out; + } + + if (entry.count == 1) { + GST_WARNING_OBJECT (self, + "This device has fixed focal length(%.2f).", entry.data.f[0]); + goto out; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + /* FIXME: Do we need to set nearest value? */ + ACaptureRequest_setEntry_float (self->capture_request, + ACAMERA_LENS_FOCAL_LENGTH, 1, &zoom); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + ret = TRUE; +out: + g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + + return ret; +} + +static GstCaps * +gst_ahc2_src_fixate (GstBaseSrc * src, GstCaps * caps) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + GstStructure *s = gst_caps_get_structure (caps, 0); + + GST_DEBUG_OBJECT (self, "Fixating : %" GST_PTR_FORMAT, caps); + + caps = gst_caps_make_writable (caps); + + /* Width/height will be fixed already here, format will + * be left for fixation by the default handler. + * We only have to fixate framerate here, to the + * highest possible framerate. + */ + gst_structure_fixate_field_nearest_fraction (s, "framerate", G_MAXINT, 1); + + caps = GST_BASE_SRC_CLASS (parent_class)->fixate (src, caps); + + return caps; +} + +static gboolean +gst_ahc2_src_set_caps (GstBaseSrc * src, GstCaps * caps) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + + GstStructure *s; + GstVideoFormat format; + gint fmt = -1; + const gchar *format_str = NULL; + const GValue *framerate_value; + gint width, height, fps_min = 0, fps_max = 0; + + ANativeWindow *image_reader_window = NULL; + + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + + GST_DEBUG_OBJECT (self, "%" GST_PTR_FORMAT, caps); + + s = gst_caps_get_structure (caps, 0); + format_str = gst_structure_get_string (s, "format"); + format = gst_video_format_from_string (format_str); + + gst_structure_get_int (s, "width", &width); + gst_structure_get_int (s, "height", &height); + framerate_value = gst_structure_get_value (s, "framerate"); + + if (GST_VALUE_HOLDS_FRACTION_RANGE (framerate_value)) { + const GValue *min, *max; + + min = gst_value_get_fraction_range_min (framerate_value); + max = gst_value_get_fraction_range_max (framerate_value); + + fps_min = + gst_value_get_fraction_numerator (min) / + gst_value_get_fraction_denominator (min); + fps_max = + gst_value_get_fraction_numerator (max) / + gst_value_get_fraction_denominator (max); + + } else if (GST_VALUE_HOLDS_FRACTION (framerate_value)) { + fps_min = fps_max = + gst_value_get_fraction_numerator (framerate_value) / + gst_value_get_fraction_denominator (framerate_value); + } else { + GST_WARNING_OBJECT (self, "framerate holds unrecognizable value"); + } + + if (fps_min != 0 && fps_max != 0) { + gint fps_range[2] = { fps_min, fps_max }; + ACaptureRequest_setEntry_i32 (self->capture_request, + ACAMERA_CONTROL_AE_TARGET_FPS_RANGE, 2, fps_range); + GST_DEBUG_OBJECT (self, "setting fps range [%d, %d]", fps_min, fps_max); + } + + switch (format) { + case GST_VIDEO_FORMAT_NV12: + fmt = AIMAGE_FORMAT_YUV_420_888; + break; + default: + GST_WARNING_OBJECT (self, "unsupported video format (%s)", format_str); + goto failed; + } + + g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); + + if (AImageReader_new (width, height, fmt, self->max_images, + &self->image_reader) != AMEDIA_OK) { + GST_ERROR_OBJECT (self, "One or more of parameter " + "(width: %d, height: %d, format: %s, max-images: %d) " + "is not supported.", width, height, format_str, self->max_images); + + goto failed; + } + + if (AImageReader_getWindow (self->image_reader, + &image_reader_window) != AMEDIA_OK) { + GST_WARNING_OBJECT (self, "Failed to get AImageReader surface"); + goto failed; + } + + AImageReader_setImageListener (self->image_reader, + &self->image_reader_listener); + + if (self->camera_output_target != NULL) { + ACaptureRequest_removeTarget (self->capture_request, + self->camera_output_target); + } + g_clear_pointer (&self->camera_output_target, + (GDestroyNotify) ACameraOutputTarget_free); + if (ACameraOutputTarget_create (image_reader_window, + &self->camera_output_target) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to create ACameraOuputTarget"); + goto failed; + } + + if (self->capture_sout != NULL) { + ACaptureSessionOutputContainer_remove (self->capture_sout_container, + self->capture_sout); + } + g_clear_pointer (&self->capture_sout, + (GDestroyNotify) ACaptureSessionOutput_free); + if (ACaptureSessionOutput_create (image_reader_window, + &self->capture_sout) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to create ACaptureSessionOutput"); + goto failed; + } + + ACaptureRequest_addTarget (self->capture_request, self->camera_output_target); + ACaptureSessionOutputContainer_add (self->capture_sout_container, + self->capture_sout); + + ACameraDevice_createCaptureSession (self->camera_device, + self->capture_sout_container, &self->session_state_cb, + &self->camera_capture_session); + + GST_DEBUG_OBJECT (self, "Starting capture request"); + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; + +failed: + g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); + + g_clear_pointer (&self->camera_output_target, + (GDestroyNotify) ACameraOutputTarget_free); + + return FALSE; +} + +static GstCaps * +gst_ahc2_src_get_caps (GstBaseSrc * src, GstCaps * filter) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + GstCaps *caps = gst_caps_new_empty (); + GstStructure *format = NULL; + + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry, fps_entry; + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + + GST_ERROR_OBJECT (self, "Failed to get metadata from camera device"); + + return caps; + } + + GST_OBJECT_LOCK (self); + +#ifndef GST_DISABLE_GST_DEBUG + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL, &entry) == ACAMERA_OK) { + guint8 level = entry.data.u8[0]; + gchar *level_str = NULL; + switch (level) { + case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED: + level_str = g_strdup ("limited"); + break; + case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_FULL: + level_str = g_strdup ("full"); + break; + case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY: + level_str = g_strdup ("legacy"); + break; + case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_3: + level_str = g_strdup ("3"); + break; + default: + level_str = g_strdup ("unknown"); + GST_WARNING_OBJECT (self, "Unknown supported hardware level (%d)", + level); + break; + } + + GST_DEBUG_OBJECT (self, "Hardware supported level: %s", level_str); + g_free (level_str); + } + + GST_DEBUG_OBJECT (self, "List of capabilities:"); + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_REQUEST_AVAILABLE_CAPABILITIES, &entry) == ACAMERA_OK) { + int i = 0; + for (; i < entry.count; i++) { + guint8 cap = entry.data.u8[i]; + gchar *cap_str = NULL; + switch (cap) { + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE: + cap_str = g_strdup (" backward compatible"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_MANUAL_SENSOR: + cap_str = g_strdup (" manual sensor"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_MANUAL_POST_PROCESSING: + cap_str = g_strdup (" manual post processing"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_RAW: + cap_str = g_strdup (" raw"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_READ_SENSOR_SETTINGS: + cap_str = g_strdup (" read sensor settings"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_BURST_CAPTURE: + cap_str = g_strdup (" burst capture"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT: + cap_str = g_strdup (" depth output"); + break; + default: + cap_str = g_strdup (" unknown"); + break; + } + GST_DEBUG_OBJECT (self, " %s(%u)", cap_str, cap); + g_free (cap_str); + } + } +#endif + + GST_DEBUG_OBJECT (self, "Checking available FPS ranges:"); + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES, + &fps_entry) == ACAMERA_OK) { + int i = 0; + for (; i < fps_entry.count; i += 2) { + GST_DEBUG_OBJECT (self, " (min: %d, max: %d)", fps_entry.data.i32[i], + fps_entry.data.i32[i + 1]); + } + } + + /* Only NV12 is acceptable. */ + format = gst_structure_new ("video/x-raw", + "format", G_TYPE_STRING, "NV12", NULL); + + GST_DEBUG_OBJECT (self, "Supported available stream configurations:"); + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, + &entry) == ACAMERA_OK) { + gint i = 0; + for (; i < entry.count; i += 4) { + gint input = entry.data.i32[i + 3]; + gint fmt = entry.data.i32[i + 0]; + + if (input) + continue; + + if (fmt == AIMAGE_FORMAT_YUV_420_888) { + gint fps_idx = 0; + gint width = entry.data.i32[i + 1]; + gint height = entry.data.i32[i + 2]; + GstStructure *size = gst_structure_copy (format); + + GST_DEBUG_OBJECT (self, " (w: %d, h: %d)", width, height); + + gst_structure_set (size, + "width", G_TYPE_INT, width, + "height", G_TYPE_INT, height, + "interlaced", G_TYPE_BOOLEAN, FALSE, + "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL); + + for (fps_idx = 0; fps_idx < fps_entry.count; fps_idx += 2) { + GstStructure *s = gst_structure_copy (size); + + if (fps_entry.data.i32[fps_idx] == fps_entry.data.i32[fps_idx + 1]) { + gst_structure_set (s, "framerate", GST_TYPE_FRACTION, + fps_entry.data.i32[fps_idx], 1, NULL); + } else { + gst_structure_set (s, "framerate", GST_TYPE_FRACTION_RANGE, + fps_entry.data.i32[fps_idx], 1, fps_entry.data.i32[fps_idx + 1], + 1, NULL); + } + gst_caps_append_structure (caps, s); + } + gst_structure_free (size); + } + } + } else { + GST_WARNING_OBJECT (self, " No available stream configurations detected"); + } + + g_clear_pointer (&format, gst_structure_free); + g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + + GST_DEBUG_OBJECT (self, "%" GST_PTR_FORMAT, caps); + + GST_OBJECT_UNLOCK (self); + + return caps; +} + +static gboolean +gst_ahc2_src_unlock (GstBaseSrc * src) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + + GST_DEBUG_OBJECT (self, "unlocking create"); + gst_data_queue_set_flushing (self->outbound_queue, TRUE); + + return TRUE; +} + +static gboolean +gst_ahc2_src_unlock_stop (GstBaseSrc * src) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + + GST_DEBUG_OBJECT (self, "stopping unlock"); + gst_data_queue_set_flushing (self->outbound_queue, FALSE); + + return TRUE; +} + +static GstFlowReturn +gst_ahc2_src_create (GstPushSrc * src, GstBuffer ** buffer) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + GstDataQueueItem *item; + + if (!gst_data_queue_pop (self->outbound_queue, &item)) { + GST_DEBUG_OBJECT (self, "We're flushing"); + return GST_FLOW_FLUSHING; + } + + *buffer = GST_BUFFER (item->object); + g_free (item); + + return GST_FLOW_OK; +} + +static void +gst_ahc2_src_camera_close (GstAHC2Src * self) +{ + GST_DEBUG_OBJECT (self, "Closing Camera"); + + g_clear_pointer (&self->capture_sout, + (GDestroyNotify) ACaptureSessionOutput_free); + + g_clear_pointer (&self->camera_output_target, + (GDestroyNotify) ACameraOutputTarget_free); + + g_clear_pointer (&self->capture_request, + (GDestroyNotify) ACaptureRequest_free); + + g_clear_pointer (&self->capture_sout_container, + (GDestroyNotify) ACaptureSessionOutputContainer_free); + + g_clear_pointer (&self->camera_device, (GDestroyNotify) ACameraDevice_close); +} + +static gboolean +gst_ahc2_src_camera_open (GstAHC2Src * self) +{ + gboolean ret = FALSE; + const gchar *camera_id; + + g_return_val_if_fail (self->camera_id_list != NULL, FALSE); + + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + + camera_id = self->camera_id_list->cameraIds[self->camera_index]; + + GST_DEBUG_OBJECT (self, "Trying to open camera device (id: %s)", camera_id); + + if (ACameraManager_openCamera (self->camera_manager, camera_id, + &self->device_state_cb, &self->camera_device) != ACAMERA_OK) { + + GST_ERROR_OBJECT (self, "Failed to open camera device (id: %s)", camera_id); + goto out; + } + + if (ACameraDevice_createCaptureRequest (self->camera_device, + self->camera_template_type, &self->capture_request) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to create camera request (camera id: %s)", + camera_id); + goto out; + } + + if (ACaptureSessionOutputContainer_create (&self->capture_sout_container) != + ACAMERA_OK) { + GST_ERROR_OBJECT (self, + "Failed to create capture session output container (camera id: %s)", + camera_id); + goto out; + } + + ret = TRUE; + +out: + + if (!ret) + gst_ahc2_src_camera_close (self); + + return ret; +} + +static gboolean +gst_ahc2_src_start (GstBaseSrc * src) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + g_mutex_lock (&self->mutex); + + if (!gst_ahc2_src_camera_open (self)) { + goto out; + } + + self->previous_ts = GST_CLOCK_TIME_NONE; + self->started = TRUE; + +out: + g_mutex_unlock (&self->mutex); + + return self->started; +} + +static gboolean +gst_ahc2_src_stop (GstBaseSrc * src) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + + g_mutex_lock (&self->mutex); + ACameraCaptureSession_abortCaptures (self->camera_capture_session); + gst_ahc2_src_camera_close (self); + + gst_data_queue_flush (self->outbound_queue); + + self->started = FALSE; + g_mutex_unlock (&self->mutex); + + return TRUE; +} + +static void +gst_ahc2_src_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstAHC2Src *self = GST_AHC2_SRC (object); + + switch ((GstAHC2SrcProperty) prop_id) { + case PROP_CAMERA_INDEX: + self->camera_index = g_value_get_int (value); + break; + case PROP_CAMERA_TEMPLATE_TYPE: + self->camera_template_type = g_value_get_int (value); + break; + case PROP_MAX_IMAGES: + self->max_images = g_value_get_int (value); + break; + case PROP_COLOR_TONE_MODE:{ + GstPhotographyColorToneMode tone = g_value_get_enum (value); + gst_ahc2_src_set_color_tone_mode (GST_PHOTOGRAPHY (self), tone); + break; + } + case PROP_EV_COMPENSATION:{ + gfloat ev = g_value_get_float (value); + gst_ahc2_src_set_ev_compensation (GST_PHOTOGRAPHY (self), ev); + break; + } + case PROP_SCENE_MODE:{ + GstPhotographySceneMode scene = g_value_get_enum (value); + gst_ahc2_src_set_scene_mode (GST_PHOTOGRAPHY (self), scene); + break; + } + case PROP_WHITE_BALANCE_MODE:{ + GstPhotographyWhiteBalanceMode wb = g_value_get_enum (value); + gst_ahc2_src_set_white_balance_mode (GST_PHOTOGRAPHY (self), wb); + break; + } + case PROP_FLASH_MODE:{ + GstPhotographyFlashMode flash = g_value_get_enum (value); + gst_ahc2_src_set_flash_mode (GST_PHOTOGRAPHY (self), flash); + break; + } + case PROP_FLICKER_MODE:{ + GstPhotographyFlickerReductionMode flicker = g_value_get_enum (value); + gst_ahc2_src_set_flicker_mode (GST_PHOTOGRAPHY (self), flicker); + break; + } + case PROP_FOCUS_MODE:{ + GstPhotographyFocusMode focus = g_value_get_enum (value); + gst_ahc2_src_set_focus_mode (GST_PHOTOGRAPHY (self), focus); + break; + } + case PROP_ZOOM:{ + gfloat zoom = g_value_get_float (value); + gst_ahc2_src_set_zoom (GST_PHOTOGRAPHY (self), zoom); + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_ahc2_src_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstAHC2Src *self = GST_AHC2_SRC (object); + + switch ((GstAHC2SrcProperty) prop_id) { + case PROP_CAMERA_INDEX: + g_value_set_int (value, self->camera_index); + break; + case PROP_N_CAMERAS: + if (self->camera_id_list != NULL) { + g_value_set_int (value, self->camera_id_list->numCameras); + } else { + GST_WARNING_OBJECT (self, "Camera list doesn't exist."); + g_value_set_int (value, 0); + } + break; + case PROP_CAMERA_TEMPLATE_TYPE: + g_value_set_int (value, self->camera_template_type); + break; + case PROP_MAX_IMAGES: + g_value_set_int (value, self->max_images); + break; + case PROP_CAPABILITIES:{ + GstPhotographyCaps caps; + + caps = gst_ahc2_src_get_capabilities (GST_PHOTOGRAPHY (self)); + g_value_set_ulong (value, caps); + + break; + } + case PROP_COLOR_TONE_MODE:{ + GstPhotographyColorToneMode tone; + if (gst_ahc2_src_get_color_tone_mode (GST_PHOTOGRAPHY (self), &tone)) + g_value_set_enum (value, tone); + break; + } + case PROP_EV_COMPENSATION:{ + gfloat ev; + + if (gst_ahc2_src_get_ev_compensation (GST_PHOTOGRAPHY (self), &ev)) + g_value_set_float (value, ev); + + break; + } + case PROP_WHITE_BALANCE_MODE:{ + GstPhotographyWhiteBalanceMode wb; + if (gst_ahc2_src_get_white_balance_mode (GST_PHOTOGRAPHY (self), &wb)) + g_value_set_enum (value, wb); + break; + } + case PROP_FLASH_MODE:{ + GstPhotographyFlashMode flash; + if (gst_ahc2_src_get_flash_mode (GST_PHOTOGRAPHY (self), &flash)) + g_value_set_enum (value, flash); + break; + } + case PROP_FLICKER_MODE:{ + GstPhotographyFlickerReductionMode flicker; + if (gst_ahc2_src_get_flicker_mode (GST_PHOTOGRAPHY (self), &flicker)) + g_value_set_enum (value, flicker); + break; + } + case PROP_FOCUS_MODE:{ + GstPhotographyFocusMode focus; + if (gst_ahc2_src_get_focus_mode (GST_PHOTOGRAPHY (self), &focus)) + g_value_set_enum (value, focus); + break; + } + case PROP_SCENE_MODE:{ + GstPhotographySceneMode scene; + if (gst_ahc2_src_get_scene_mode (GST_PHOTOGRAPHY (self), &scene)) + g_value_set_enum (value, scene); + break; + } + case PROP_ZOOM:{ + gfloat zoom; + if (gst_ahc2_src_get_zoom (GST_PHOTOGRAPHY (self), &zoom)) + g_value_set_float (value, zoom); + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_ahc2_src_finalize (GObject * object) +{ + GstAHC2Src *self = GST_AHC2_SRC (object); + + g_clear_pointer (&self->camera_id_list, + (GDestroyNotify) ACameraManager_deleteCameraIdList); + + g_clear_pointer (&self->camera_manager, + (GDestroyNotify) ACameraManager_delete); + + g_clear_pointer (&self->outbound_queue, gst_object_unref); + + g_mutex_clear (&self->mutex); + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +data_queue_item_free (GstDataQueueItem * item) +{ + g_clear_pointer (&item->object, (GDestroyNotify) gst_mini_object_unref); + g_free (item); +} + +static void +image_reader_on_image_available (void *context, AImageReader * reader) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GstBuffer *buffer; + GstDataQueueItem *item; + GstClockTime duration = GST_CLOCK_TIME_NONE; + GstClockTime current_ts = GST_CLOCK_TIME_NONE; + GstClock *clock; + + GstWrappedAImage *wrapped_aimage; + AImage *image = NULL; + gint n_planes; + gint i; + + if ((clock = GST_ELEMENT_CLOCK (self))) { + GstClockTime base_time = GST_ELEMENT_CAST (self)->base_time; + gst_object_ref (clock); + current_ts = gst_clock_get_time (clock) - base_time; + gst_object_unref (clock); + } + + if (AImageReader_acquireLatestImage (reader, &image) != AMEDIA_OK) { + GST_DEBUG_OBJECT (self, "No image available"); + return; + } + g_mutex_lock (&self->mutex); + + GST_DEBUG_OBJECT (self, "Acquired an image (ts: %" GST_TIME_FORMAT ")", + GST_TIME_ARGS (current_ts)); + +#ifndef GST_DISABLE_GST_DEBUG + { + gint64 image_ts; + /* AImageReader has its own clock. */ + AImage_getTimestamp (image, &image_ts); + GST_DEBUG_OBJECT (self, " AImage internal ts: %" GST_TIME_FORMAT ")", + GST_TIME_ARGS (image_ts)); + } +#endif + + if (!self->started || GST_CLOCK_TIME_IS_VALID (self->previous_ts)) { + duration = current_ts - self->previous_ts; + self->previous_ts = current_ts; + } else { + self->previous_ts = current_ts; + /* Dropping first image to calculate duration */ + AImage_delete (image); + GST_DEBUG_OBJECT (self, "Droping image (reason: %s)", + self->started ? "first frame" : "not yet started"); + g_mutex_unlock (&self->mutex); + return; + } + + AImage_getNumberOfPlanes (image, &n_planes); + + buffer = gst_buffer_new (); + GST_BUFFER_DURATION (buffer) = duration; + GST_BUFFER_PTS (buffer) = current_ts; + + wrapped_aimage = g_new0 (GstWrappedAImage, 1); + wrapped_aimage->refcount = 1; + wrapped_aimage->ahc2src = g_object_ref (self); + wrapped_aimage->image = image; + + for (i = 0; i < n_planes; i++) { + guint8 *data; + gint length; + GstMemory *mem; + + AImage_getPlaneData (image, i, &data, &length); + + mem = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, + data, length, 0, length, + gst_wrapped_aimage_ref (wrapped_aimage), + (GDestroyNotify) gst_wrapped_aimage_unref); + + GST_TRACE_OBJECT (self, "Created a wrapped memory (ptr: %p, length: %d)", + mem, length); + gst_buffer_append_memory (buffer, mem); + } + + item = g_new0 (GstDataQueueItem, 1); + item->object = GST_MINI_OBJECT (buffer); + item->size = gst_buffer_get_size (buffer); + item->visible = TRUE; + item->destroy = (GDestroyNotify) data_queue_item_free; + + if (!gst_data_queue_push (self->outbound_queue, item)) { + item->destroy (item); + GST_DEBUG_OBJECT (self, "Failed to push item because we're flushing"); + } + + gst_wrapped_aimage_unref (wrapped_aimage); + g_mutex_unlock (&self->mutex); + + GST_TRACE_OBJECT (self, + "created buffer from image callback %" G_GSIZE_FORMAT ", ts %" + GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT + ", offset %" G_GINT64_FORMAT ", offset_end %" G_GINT64_FORMAT, + gst_buffer_get_size (buffer), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer)), + GST_TIME_ARGS (GST_BUFFER_DURATION (buffer)), + GST_BUFFER_OFFSET (buffer), GST_BUFFER_OFFSET_END (buffer)); +} + +static void +camera_device_on_disconnected (void *context, ACameraDevice * device) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GST_DEBUG_OBJECT (self, "Camera[%s] is disconnected", + ACameraDevice_getId (device)); +} + +static void +camera_device_on_error (void *context, ACameraDevice * device, int error) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GST_ERROR_OBJECT (self, "Error (code: %d) on Camera[%s]", error, + ACameraDevice_getId (device)); +} + +static void +capture_session_on_ready (void *context, ACameraCaptureSession * session) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GST_DEBUG_OBJECT (self, "Camera[%s] capture session is ready", + ACameraDevice_getId (self->camera_device)); +} + +static void +capture_session_on_active (void *context, ACameraCaptureSession * session) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GST_DEBUG_OBJECT (self, "Camera[%s] capture session is activated", + ACameraDevice_getId (self->camera_device)); +} + +static const gchar * +gst_ahc2_src_get_camera_id_by_index (GstAHC2Src * self, gint idx) +{ + g_return_val_if_fail (self->camera_id_list != NULL, NULL); + g_return_val_if_fail (idx >= self->camera_id_list->numCameras, NULL); + + return self->camera_id_list->cameraIds[idx]; +} + +static void +gst_ahc2_src_class_init (GstAHC2SrcClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass); + GstBaseSrcClass *gstbasesrc_class = GST_BASE_SRC_CLASS (klass); + GstPushSrcClass *gstpushsrc_class = GST_PUSH_SRC_CLASS (klass); + + gobject_class->set_property = gst_ahc2_src_set_property; + gobject_class->get_property = gst_ahc2_src_get_property; + gobject_class->finalize = gst_ahc2_src_finalize; + + /** + * GstAHC2Src:camera-index: + * + * The Camera index to be connected. + */ + properties[PROP_CAMERA_INDEX] = + g_param_spec_int ("camera-index", "Camera Index", + "The camera device index to open", 0, G_MAXINT, 0, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS); + + /** + * GstAHC2Src:n-cameras: + * + * The number of connected camera devices. + */ + properties[PROP_N_CAMERAS] = + g_param_spec_int ("n-cameras", "Number of Cameras", + "The number of connected camera devices", 0, G_MAXINT, 0, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS); + + /** + * GstAHC2Src:camera-template-type: + * The suitable template type for the purpose of running camera. + */ + properties[PROP_CAMERA_TEMPLATE_TYPE] = + g_param_spec_int ("camera-template-type", "Camera Template Type", + "A request template for the camera running purpose", -1, G_MAXINT, + TEMPLATE_PREVIEW, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS); + + /** + * GstAHC2Src:max-images: + * The number of buffering images. + */ + properties[PROP_MAX_IMAGES] = + g_param_spec_int ("max-images", "Max Images", + "The number of buffering images", 0, G_MAXINT, + DEFAULT_MAX_IMAGES, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_PAUSED | G_PARAM_STATIC_STRINGS); + + g_object_class_install_properties (gobject_class, PROP_N_INSTALL, properties); + + /* Override GstPhotography properties */ + g_object_class_override_property (gobject_class, PROP_ANALOG_GAIN, + GST_PHOTOGRAPHY_PROP_ANALOG_GAIN); + properties[PROP_ANALOG_GAIN] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_ANALOG_GAIN); + + g_object_class_override_property (gobject_class, PROP_APERTURE, + GST_PHOTOGRAPHY_PROP_APERTURE); + properties[PROP_APERTURE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_APERTURE); + + g_object_class_override_property (gobject_class, PROP_CAPABILITIES, + GST_PHOTOGRAPHY_PROP_CAPABILITIES); + properties[PROP_CAPABILITIES] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_CAPABILITIES); + + g_object_class_override_property (gobject_class, PROP_COLOR_TEMPERATURE, + GST_PHOTOGRAPHY_PROP_COLOR_TEMPERATURE); + properties[PROP_COLOR_TEMPERATURE] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_COLOR_TEMPERATURE); + + g_object_class_override_property (gobject_class, PROP_COLOR_TONE_MODE, + GST_PHOTOGRAPHY_PROP_COLOR_TONE); + properties[PROP_COLOR_TONE_MODE] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_COLOR_TONE); + + g_object_class_override_property (gobject_class, PROP_EV_COMPENSATION, + GST_PHOTOGRAPHY_PROP_EV_COMP); + properties[PROP_EV_COMPENSATION] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_EV_COMP); + + g_object_class_override_property (gobject_class, PROP_EXPOSURE_TIME, + GST_PHOTOGRAPHY_PROP_EXPOSURE_TIME); + properties[PROP_EXPOSURE_TIME] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_EXPOSURE_TIME); + + g_object_class_override_property (gobject_class, PROP_FLASH_MODE, + GST_PHOTOGRAPHY_PROP_FLASH_MODE); + properties[PROP_FLASH_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_FLASH_MODE); + + g_object_class_override_property (gobject_class, PROP_FLICKER_MODE, + GST_PHOTOGRAPHY_PROP_FLICKER_MODE); + properties[PROP_FLICKER_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_FLICKER_MODE); + + g_object_class_override_property (gobject_class, PROP_FOCUS_MODE, + GST_PHOTOGRAPHY_PROP_FOCUS_MODE); + properties[PROP_FOCUS_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_FOCUS_MODE); + + g_object_class_override_property (gobject_class, + PROP_IMAGE_CAPTURE_SUPPORTED_CAPS, + GST_PHOTOGRAPHY_PROP_IMAGE_CAPTURE_SUPPORTED_CAPS); + properties[PROP_IMAGE_CAPTURE_SUPPORTED_CAPS] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_IMAGE_CAPTURE_SUPPORTED_CAPS); + + g_object_class_override_property (gobject_class, + PROP_IMAGE_PREVIEW_SUPPORTED_CAPS, + GST_PHOTOGRAPHY_PROP_IMAGE_PREVIEW_SUPPORTED_CAPS); + properties[PROP_IMAGE_PREVIEW_SUPPORTED_CAPS] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_IMAGE_PREVIEW_SUPPORTED_CAPS); + + g_object_class_override_property (gobject_class, PROP_ISO_SPEED, + GST_PHOTOGRAPHY_PROP_ISO_SPEED); + properties[PROP_ISO_SPEED] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_ISO_SPEED); + + g_object_class_override_property (gobject_class, PROP_LENS_FOCUS, + GST_PHOTOGRAPHY_PROP_LENS_FOCUS); + properties[PROP_LENS_FOCUS] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_LENS_FOCUS); + + g_object_class_override_property (gobject_class, PROP_MAX_EXPOSURE_TIME, + GST_PHOTOGRAPHY_PROP_MAX_EXPOSURE_TIME); + properties[PROP_MAX_EXPOSURE_TIME] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_MAX_EXPOSURE_TIME); + + g_object_class_override_property (gobject_class, PROP_MIN_EXPOSURE_TIME, + GST_PHOTOGRAPHY_PROP_MIN_EXPOSURE_TIME); + properties[PROP_MIN_EXPOSURE_TIME] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_MIN_EXPOSURE_TIME); + + g_object_class_override_property (gobject_class, PROP_NOISE_REDUCTION, + GST_PHOTOGRAPHY_PROP_NOISE_REDUCTION); + properties[PROP_NOISE_REDUCTION] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_NOISE_REDUCTION); + + g_object_class_override_property (gobject_class, PROP_SCENE_MODE, + GST_PHOTOGRAPHY_PROP_SCENE_MODE); + properties[PROP_SCENE_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_SCENE_MODE); + + g_object_class_override_property (gobject_class, PROP_WHITE_BALANCE_MODE, + GST_PHOTOGRAPHY_PROP_WB_MODE); + properties[PROP_WHITE_BALANCE_MODE] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_WB_MODE); + + g_object_class_override_property (gobject_class, PROP_WHITE_POINT, + GST_PHOTOGRAPHY_PROP_WHITE_POINT); + properties[PROP_WHITE_POINT] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_WHITE_POINT); + + g_object_class_override_property (gobject_class, PROP_ZOOM, + GST_PHOTOGRAPHY_PROP_ZOOM); + properties[PROP_ZOOM] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_ZOOM); + + signals[SIG_GET_CAMERA_ID_BY_INDEX] = + g_signal_new ("get-camera-id-by-index", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (GstAHC2SrcClass, + get_camera_id_by_index), NULL, NULL, NULL, + G_TYPE_CHAR, 1, G_TYPE_INT); + + gst_element_class_add_static_pad_template (gstelement_class, &src_template); + gst_element_class_set_static_metadata (gstelement_class, + "Android Camera2 Source", "Source/Video/Device", + "Source element for Android hardware camers 2 API", + "Justin Kim "); + + gstbasesrc_class->get_caps = GST_DEBUG_FUNCPTR (gst_ahc2_src_get_caps); + gstbasesrc_class->set_caps = GST_DEBUG_FUNCPTR (gst_ahc2_src_set_caps); + gstbasesrc_class->fixate = GST_DEBUG_FUNCPTR (gst_ahc2_src_fixate); + gstbasesrc_class->start = GST_DEBUG_FUNCPTR (gst_ahc2_src_start); + gstbasesrc_class->stop = GST_DEBUG_FUNCPTR (gst_ahc2_src_stop); + gstbasesrc_class->unlock = GST_DEBUG_FUNCPTR (gst_ahc2_src_unlock); + gstbasesrc_class->unlock_stop = GST_DEBUG_FUNCPTR (gst_ahc2_src_unlock_stop); + + gstpushsrc_class->create = GST_DEBUG_FUNCPTR (gst_ahc2_src_create); + + klass->get_camera_id_by_index = gst_ahc2_src_get_camera_id_by_index; + + GST_DEBUG_CATEGORY_INIT (gst_debug_ahc2_src, "ahc2src", 0, "AHC2 Source"); +} + +static gboolean +data_queue_check_full_cb (GstDataQueue * queue, guint visible, + guint bytes, guint64 time, gpointer checkdata) +{ + return FALSE; +} + +static void +gst_ahc2_src_init (GstAHC2Src * self) +{ + self->camera_template_type = TEMPLATE_PREVIEW; + self->max_images = DEFAULT_MAX_IMAGES; + self->outbound_queue = + gst_data_queue_new (data_queue_check_full_cb, NULL, NULL, NULL); + + gst_base_src_set_format (GST_BASE_SRC (self), GST_FORMAT_TIME); + gst_base_src_set_live (GST_BASE_SRC (self), TRUE); + + self->camera_manager = ACameraManager_create (); + + if (ACameraManager_getCameraIdList (self->camera_manager, + &self->camera_id_list) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get camera id list"); + goto fail; + } + + if (self->camera_id_list->numCameras < 1) { + GST_WARNING_OBJECT (self, "No camera device detected."); + goto fail; + } + + self->device_state_cb.context = self; + self->device_state_cb.onDisconnected = camera_device_on_disconnected; + self->device_state_cb.onError = camera_device_on_error; + + self->session_state_cb.context = self; + self->session_state_cb.onReady = capture_session_on_ready; + self->session_state_cb.onActive = capture_session_on_active; + + self->image_reader_listener.context = self; + self->image_reader_listener.onImageAvailable = + image_reader_on_image_available; + + g_mutex_init (&self->mutex); + +#ifndef GST_DISABLE_GST_DEBUG + { + int idx = 0; + + GST_DEBUG_OBJECT (self, "detected %d cameras", + self->camera_id_list->numCameras); + for (; idx < self->camera_id_list->numCameras; idx++) { + GST_DEBUG_OBJECT (self, " camera device[%d]: %s", idx, + self->camera_id_list->cameraIds[idx]); + } + } +#endif + + return; + +fail: + g_clear_pointer (&self->camera_id_list, + (GDestroyNotify) ACameraManager_deleteCameraIdList); + + g_clear_pointer (&self->camera_manager, + (GDestroyNotify) ACameraManager_delete); +} + +static void +gst_ahc2_src_photography_init (gpointer g_iface, gpointer iface_data) +{ + GstPhotographyInterface *iface = g_iface; + + iface->get_capabilities = gst_ahc2_src_get_capabilities; + + iface->get_color_tone_mode = gst_ahc2_src_get_color_tone_mode; + iface->set_color_tone_mode = gst_ahc2_src_set_color_tone_mode; + + iface->set_autofocus = gst_ahc2_src_set_autofocus; + iface->get_focus_mode = gst_ahc2_src_get_focus_mode; + iface->set_focus_mode = gst_ahc2_src_set_focus_mode; + + iface->get_flash_mode = gst_ahc2_src_get_flash_mode; + iface->set_flash_mode = gst_ahc2_src_set_flash_mode; + + iface->get_flicker_mode = gst_ahc2_src_get_flicker_mode; + iface->set_flicker_mode = gst_ahc2_src_set_flicker_mode; + + iface->get_scene_mode = gst_ahc2_src_get_scene_mode; + iface->set_scene_mode = gst_ahc2_src_set_scene_mode; + + iface->get_white_balance_mode = gst_ahc2_src_get_white_balance_mode; + iface->set_white_balance_mode = gst_ahc2_src_set_white_balance_mode; + + iface->get_ev_compensation = gst_ahc2_src_get_ev_compensation; + iface->set_ev_compensation = gst_ahc2_src_set_ev_compensation; + + iface->get_zoom = gst_ahc2_src_get_zoom; + iface->set_zoom = gst_ahc2_src_set_zoom; +} diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.h b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.h new file mode 100644 index 00000000000..f222ffb98c4 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.h @@ -0,0 +1,56 @@ +/* GStreamer android.hardware.Camera2 Source + * Copyright (C) 2017, Collabora Ltd. + * Author:Justin Kim + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_AHC2_SRC_H__ +#define __GST_AHC2_SRC_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_AHC2_SRC (gst_ahc2_src_get_type ()) +#define GST_IS_AHC2_SRC(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_AHC2_SRC)) +#define GST_IS_AHC2_SRC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_AHC2_SRC)) +#define GST_AHC2_SRC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_AHC2_SRC, GstAHC2SrcClass)) +#define GST_AHC2_SRC(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_AHC2_SRC, GstAHC2Src)) +#define GST_AHC2_SRC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_AHC2_SRC, GstAHC2SrcClass)) +#define GST_AHC2_SRC_CAST(obj) ((GstAHC2Src*)(obj)) +#define GST_AHC2_SRC_CLASS_CAST(klass) ((GstAHC2SrcClass*)(klass)) + +typedef struct _GstAHC2Src GstAHC2Src; +typedef struct _GstAHC2SrcClass GstAHC2SrcClass; +typedef struct _GstAHC2SrcPrivate GstAHC2SrcPrivate; + +struct _GstAHC2SrcClass { + GstPushSrcClass parent_class; + + /* actions */ + const gchar* (*get_camera_id_by_index) (GstAHC2Src *self, gint idx); + + gpointer _gst_reserved[GST_PADDING_LARGE]; +}; + +GST_EXPORT +GType gst_ahc2_src_get_type (void); + +G_END_DECLS + +#endif /* __GST_AHC2_SRC_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c index 2da4657bd54..ba525878a3b 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c @@ -35,6 +35,8 @@ #include "gstjniutils.h" #endif +#include "gstahc2src.h" + #include "gstamc.h" #include "gstamc-constants.h" @@ -1932,6 +1934,13 @@ plugin_init (GstPlugin * plugin) init_ok = TRUE; #endif + if (gst_element_register (plugin, "ahc2src", GST_RANK_NONE, + GST_TYPE_AHC2_SRC)) { + init_ok = TRUE; + } else { + GST_ERROR ("Failed to register android camera2 source"); + } + return init_ok; } diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/meson.build b/subprojects/gst-plugins-bad/sys/androidmedia/meson.build index 7199faea2f7..e0a27c6769a 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/meson.build +++ b/subprojects/gst-plugins-bad/sys/androidmedia/meson.build @@ -74,7 +74,16 @@ else plugin_name = 'gstandroidmedia' endif -if have_jni_h or have_mlsdk +have_ndkcamera = cc.has_header('camera/NdkCameraError.h', required : false) +if have_ndkcamera + androidmedia_sources += ['gstahc2src.c'] + foreach lib : ['camera2ndk', 'mediandk'] + dep = cc.find_library(lib, required : true) + extra_deps += dep + endforeach +endif + +if have_jni_h or have_mlsdk or have_ndkcamera gstandroidmedia = library(plugin_name, androidmedia_sources, c_args : [gst_plugins_bad_args, extra_cargs], From 3b473609914d68503c4fb179ea0e545c6f3e2ca4 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 25 May 2022 07:39:43 -0300 Subject: [PATCH 238/339] androidmedia: added gstacamdeviceprovider Enables device enumeration and monitor using the ACameraManager API --- .../sys/androidmedia/gstacamdeviceprovider.c | 439 ++++++++++++++++++ .../sys/androidmedia/gstacamdeviceprovider.h | 44 ++ .../gst-plugins-bad/sys/androidmedia/gstamc.c | 7 + .../sys/androidmedia/meson.build | 6 +- 4 files changed, 495 insertions(+), 1 deletion(-) create mode 100644 subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c create mode 100644 subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.h diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c new file mode 100644 index 00000000000..7ace1cbd3a6 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c @@ -0,0 +1,439 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstacamdeviceprovider.h" + +#include + +struct _GstAcamDeviceProvider +{ + GstDeviceProvider parent; + + ACameraManager *manager; + + struct ACameraManager_AvailabilityListener listeners; +}; + +struct _GstAcamDevice +{ + GstDevice parent; + + gchar *camera_id; + gint camera_index; +}; + +G_DEFINE_TYPE (GstAcamDeviceProvider, gst_acam_device_provider, + GST_TYPE_DEVICE_PROVIDER); + +G_DEFINE_TYPE (GstAcamDevice, gst_acam_device, GST_TYPE_DEVICE); + +GST_DEVICE_PROVIDER_REGISTER_DEFINE (acamdeviceprovider, "acamdeviceprovider", + GST_RANK_PRIMARY, GST_TYPE_ACAM_DEVICE_PROVIDER); + +GST_DEBUG_CATEGORY_STATIC (gst_acam_device_provider_debug); +#define GST_CAT_DEFAULT (gst_acam_device_provider_debug) +#define device_provider_parent_class gst_acam_device_provider_parent_class + +#define CAM_FRONT "Front Camera" +#define CAM_BACK "Back Camera" +#define CAM_EXT "External Camera" +#define CAM_UNKNOWN "Unknown Camera" + + +static gchar * +_build_camera_display_name (gint camera_index, uint8_t lens_facing, + gfloat lens_aperture) +{ + if (camera_index == -1) + return g_strdup (CAM_UNKNOWN); + + if (lens_facing == ACAMERA_LENS_FACING_FRONT) + return g_strdup_printf ("%s (f/%.2f)", CAM_FRONT, lens_aperture); + if (lens_facing == ACAMERA_LENS_FACING_BACK) + return g_strdup_printf ("%s (f/%.2f)", CAM_BACK, lens_aperture); + if (lens_facing == ACAMERA_LENS_FACING_EXTERNAL) + return g_strdup_printf ("%s (%d, f/%.2f)", CAM_EXT, camera_index + 1, + lens_aperture); + + return g_strdup (CAM_UNKNOWN); +} + +static gchar * +gst_acam_device_provider_get_device_display_name (GstAcamDeviceProvider * + provider, const char *camera_id, gint camera_index) +{ + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + uint8_t lens_facing = 0; + gfloat lens_aperture = 0.f; + + if (ACameraManager_getCameraCharacteristics (provider->manager, + camera_id, &metadata) != ACAMERA_OK) { + + GST_ERROR ("Failed to retrieve camera (%s) metadata", camera_id); + return _build_camera_display_name (-1, 0, lens_aperture); + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_LENS_FACING, &entry) != ACAMERA_OK) { + GST_ERROR ("Failed to retrieve camera (%s) LENS_FACING", camera_id); + camera_index = -1; + } else { + lens_facing = entry.data.u8[0]; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_LENS_INFO_AVAILABLE_APERTURES, &entry) != ACAMERA_OK) { + GST_ERROR + ("Failed to retrieve camera (%s) LENS_INFO_AVAILABLE_FOCAL_LENGTHS", + camera_id); + } else { + lens_aperture = entry.data.f[0]; + } + + ACameraMetadata_free (metadata); + return _build_camera_display_name (camera_index, lens_facing, lens_aperture); +} + +static GstDevice * +gst_acam_device_provider_create_device (GstAcamDeviceProvider * + provider, const char *camera_id, gint camera_index) +{ + GstAcamDevice *device; + gchar *display_name; + GstCaps *caps; + + GST_TRACE_OBJECT (provider, + "Creating device with camera_id: %s, camera_index: %d", camera_id, + camera_index); + + display_name = + gst_acam_device_provider_get_device_display_name (provider, camera_id, + camera_index); + caps = gst_caps_new_empty_simple ("video/x-raw"); + + device = g_object_new (GST_TYPE_ACAM_DEVICE, // + "device-class", "Video/Source", // + "display-name", display_name, // + "caps", caps, // + "camera-id", g_strdup (camera_id), // + "camera-index", camera_index, // + NULL); + + return GST_DEVICE_CAST (device); +} + +static GList * +gst_acam_device_provider_probe (GstDeviceProvider * object) +{ + GList *devices = NULL; + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (object); + ACameraIdList *id_list; + gint i; + + if (ACameraManager_getCameraIdList (provider->manager, + &id_list) != ACAMERA_OK) { + GST_ERROR_OBJECT (provider, "Failed to get camera id list"); + return NULL; + } + + GST_DEBUG_OBJECT (provider, "Found %d cameras", id_list->numCameras); + + for (i = 0; i < id_list->numCameras; i++) { + const char *camera_id = id_list->cameraIds[i]; + GstDevice *device; + + device = gst_acam_device_provider_create_device (provider, camera_id, i); + + gst_object_ref_sink (device); + devices = g_list_append (devices, device); + } + + ACameraManager_deleteCameraIdList (id_list); + return devices; +} + +static gboolean +gst_acam_device_provider_start (GstDeviceProvider * object) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (object); + + GST_DEBUG_OBJECT (provider, "Starting..."); + + if (!provider->manager) { + GST_ERROR_OBJECT (provider, + "Starting without an instance of ACameraManager"); + return FALSE; + } + + if (ACameraManager_registerAvailabilityCallback (provider->manager, + &provider->listeners) != ACAMERA_OK) { + GST_ERROR_OBJECT (provider, "Couldn't register Availability Callbacks"); + return FALSE; + } + + return TRUE; +} + +static void +gst_acam_device_provider_stop (GstDeviceProvider * object) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (object); + + GST_DEBUG_OBJECT (provider, "Stopping..."); + + if (!provider->manager) { + GST_ERROR_OBJECT (provider, + "Stopping without an instance of ACameraManager"); + return; + } + + if (ACameraManager_unregisterAvailabilityCallback (provider->manager, + &provider->listeners) != ACAMERA_OK) { + GST_ERROR_OBJECT (provider, "Couldn't unregister Availability Callbacks"); + return; + } +} + +static gint +gst_acam_device_provider_find_camera_index (GstAcamDeviceProvider * + provider, const char *camera_id) +{ + gint i, idx = -1; + ACameraIdList *id_list; + + if (ACameraManager_getCameraIdList (provider->manager, + &id_list) != ACAMERA_OK) { + GST_ERROR_OBJECT (provider, "Failed to get camera id list"); + return -1; + } + + for (i = 0; i < id_list->numCameras; i++) { + const char *id = id_list->cameraIds[i]; + if (g_strcmp0 (camera_id, id) == 0) { + idx = i; + break; + } + } + + ACameraManager_deleteCameraIdList (id_list); + return idx; +} + +static void +gst_acam_device_provider_on_camera_available (void *context, + const char *camera_id) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (context); + gint camera_index; + GstDevice *device; + + camera_index = + gst_acam_device_provider_find_camera_index (provider, camera_id); + device = + gst_acam_device_provider_create_device (provider, camera_id, + camera_index); + + GST_INFO_OBJECT (provider, "Camera became available (%s)", camera_id); + gst_device_provider_device_add (GST_DEVICE_PROVIDER_CAST (provider), device); +} + +static void +gst_acam_device_provider_remove_device (GstAcamDeviceProvider * acamprovider, + const char *camera_id_to_remove) +{ + GstDeviceProvider *provider = GST_DEVICE_PROVIDER_CAST (acamprovider); + GstDevice *device = NULL; + GList *item; + + GST_OBJECT_LOCK (provider); + for (item = provider->devices; item; item = item->next) { + device = item->data; + gchar *camera_id; + gboolean found; + + g_object_get (device, "camera-id", &camera_id, NULL); + found = (g_strcmp0 (camera_id_to_remove, camera_id) == 0); + g_free (camera_id); + + if (found) { + gst_object_ref (device); + break; + } + + device = NULL; + } + GST_OBJECT_UNLOCK (provider); + + if (device) { + gst_device_provider_device_remove (provider, device); + g_object_unref (device); + } +} + + +static void +gst_acam_device_provider_on_camera_unavailable (void *context, + const char *camera_id) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (context); + ACameraMetadata *metadata = NULL; + + // if we can't retrieve the metadata for the camera, it has been disconnected + if (ACameraManager_getCameraCharacteristics (provider->manager, + camera_id, &metadata) != ACAMERA_OK) { + GST_INFO_OBJECT (provider, "Camera became unavailable (%s)", camera_id); + gst_acam_device_provider_remove_device (provider, camera_id); + } + + if (metadata) + ACameraMetadata_free (metadata); +} + +static void +gst_acam_device_provider_init (GstAcamDeviceProvider * provider) +{ + provider->manager = ACameraManager_create (); + + provider->listeners.onCameraAvailable = + gst_acam_device_provider_on_camera_available; + provider->listeners.onCameraUnavailable = + gst_acam_device_provider_on_camera_unavailable; + provider->listeners.context = provider; +} + +static void +gst_acam_device_provider_finalize (GObject * object) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (object); + + if (provider->manager) { + ACameraManager_delete (provider->manager); + provider->manager = NULL; + } + + G_OBJECT_CLASS (device_provider_parent_class)->finalize (object); +} + +static void +gst_acam_device_provider_class_init (GstAcamDeviceProviderClass * klass) +{ + GstDeviceProviderClass *dm_class = GST_DEVICE_PROVIDER_CLASS (klass); + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + dm_class->probe = gst_acam_device_provider_probe; + dm_class->start = gst_acam_device_provider_start; + dm_class->stop = gst_acam_device_provider_stop; + + gobject_class->finalize = gst_acam_device_provider_finalize; + + gst_device_provider_class_set_static_metadata (dm_class, + "Android Camera Device Provider", "Source/Video", + "List and monitor Android camera devices", + "Tulio Beloqui "); + + GST_DEBUG_CATEGORY_INIT (gst_acam_device_provider_debug, + "acamdeviceprovider", 0, "Android Camera Device Provider"); +} + +static GstElement * +gst_acam_device_create_element (GstDevice * object, const gchar * name) +{ + GstAcamDevice *device = GST_ACAM_DEVICE_CAST (object); + GstElement *elem; + + elem = gst_element_factory_make ("ahc2src", name); + g_object_set (elem, "camera-index", device->camera_index, NULL); + + return elem; +} + +enum +{ + PROP_CAMERA_INDEX = 1, + PROP_CAMERA_ID = 2, +}; + +static void +gst_acam_device_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstAcamDevice *device = GST_ACAM_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_CAMERA_INDEX: + g_value_set_int (value, device->camera_index); + break; + case PROP_CAMERA_ID: + g_value_set_string (value, (gchar *) device->camera_id); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_acam_device_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstAcamDevice *device = GST_ACAM_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_CAMERA_INDEX: + device->camera_index = g_value_get_int (value); + break; + case PROP_CAMERA_ID: + if (device->camera_id) + g_free (device->camera_id); + + device->camera_id = g_strdup (g_value_get_string (value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_acam_device_class_init (GstAcamDeviceClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstDeviceClass *dev_class = GST_DEVICE_CLASS (klass); + + dev_class->create_element = gst_acam_device_create_element; + gobject_class->get_property = gst_acam_device_get_property; + gobject_class->set_property = gst_acam_device_set_property; + + g_object_class_install_property (gobject_class, PROP_CAMERA_INDEX, + g_param_spec_int ("camera-index", "Camera Index", + "The camera device index", 0, G_MAXINT, 0, + G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); + + g_object_class_install_property (gobject_class, PROP_CAMERA_ID, + g_param_spec_string ("camera-id", "Camera ID", + "The Unique ID for the camera", + NULL, G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); +} + +static void +gst_acam_device_init (GstAcamDevice * device) +{ +} diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.h b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.h new file mode 100644 index 00000000000..d42c181b21b --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.h @@ -0,0 +1,44 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_ACAM_DEVICE_PROVIDER_H__ +#define __GST_ACAM_DEVICE_PROVIDER_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_ACAM_DEVICE_PROVIDER (gst_acam_device_provider_get_type ()) +#define GST_ACAM_DEVICE_PROVIDER_CAST(obj) ((GstAcamDeviceProvider *)(obj)) + +#define GST_TYPE_ACAM_DEVICE (gst_acam_device_get_type ()) +#define GST_ACAM_DEVICE_CAST(obj) ((GstAcamDevice *)(obj)) + +G_DECLARE_FINAL_TYPE (GstAcamDevice, gst_acam_device, GST, ACAM_DEVICE, + GstDevice) + +G_DECLARE_FINAL_TYPE (GstAcamDeviceProvider, gst_acam_device_provider, GST, + ACAM_DEVICE_PROVIDER, GstDeviceProvider) + +GST_DEVICE_PROVIDER_REGISTER_DECLARE (acamdeviceprovider); + +G_END_DECLS + +#endif /* __GST_ACAM_DEVICE_PROVIDER_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c index ba525878a3b..4f348f7f6ec 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c @@ -36,6 +36,7 @@ #endif #include "gstahc2src.h" +#include "gstacamdeviceprovider.h" #include "gstamc.h" #include "gstamc-constants.h" @@ -1941,6 +1942,12 @@ plugin_init (GstPlugin * plugin) GST_ERROR ("Failed to register android camera2 source"); } + if (GST_DEVICE_PROVIDER_REGISTER (acamdeviceprovider, plugin)) { + init_ok = TRUE; + } else { + GST_ERROR ("Failed to register android camera device provider"); + } + return init_ok; } diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/meson.build b/subprojects/gst-plugins-bad/sys/androidmedia/meson.build index e0a27c6769a..810f8c2feba 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/meson.build +++ b/subprojects/gst-plugins-bad/sys/androidmedia/meson.build @@ -76,7 +76,11 @@ endif have_ndkcamera = cc.has_header('camera/NdkCameraError.h', required : false) if have_ndkcamera - androidmedia_sources += ['gstahc2src.c'] + androidmedia_sources += [ + 'gstacamdeviceprovider.c', + 'gstahc2src.c' + ] + foreach lib : ['camera2ndk', 'mediandk'] dep = cc.find_library(lib, required : true) extra_deps += dep From 45af5dc9d1f65ee4264f2c67b452a21510839506 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 31 May 2022 15:16:11 +0200 Subject: [PATCH 239/339] gstamdeviceprovider: added audio device provider for android --- .../gst-plugins-bad/sys/androidmedia/gstamc.c | 9 + .../sys/androidmedia/gstamdeviceprovider.c | 788 ++++++++++++++++++ .../sys/androidmedia/gstamdeviceprovider.h | 44 + .../sys/androidmedia/meson.build | 9 + .../GstAmAudioDeviceCallback.java | 45 + 5 files changed, 895 insertions(+) create mode 100644 subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c create mode 100644 subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.h create mode 100644 subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c index 4f348f7f6ec..b4db71b3e28 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c @@ -32,6 +32,7 @@ #ifdef HAVE_JNI_H #include "gstahcsrc.h" #include "gstahssrc.h" +#include "gstamdeviceprovider.h" #include "gstjniutils.h" #endif @@ -1948,6 +1949,14 @@ plugin_init (GstPlugin * plugin) GST_ERROR ("Failed to register android camera device provider"); } +#ifdef HAVE_JNI_H + if (GST_DEVICE_PROVIDER_REGISTER (amdeviceprovider, plugin)) { + init_ok = TRUE; + } else { + GST_ERROR ("Failed to register android AudioManager device provider"); + } +#endif + return init_ok; } diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c new file mode 100644 index 00000000000..0d9f2b44cdd --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c @@ -0,0 +1,788 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstamdeviceprovider.h" + +#include "gstjniutils.h" + +struct _GstAmDeviceProvider +{ + GstDeviceProvider parent; + + jobject audioDeviceCB; +}; + +struct _GstAmDevice +{ + GstDevice parent; + + gint device_id; +}; + +enum +{ + ADDED_DEVICES, + REMOVED_DEVICES, +}; + +G_DEFINE_TYPE (GstAmDeviceProvider, gst_am_device_provider, + GST_TYPE_DEVICE_PROVIDER); + +G_DEFINE_TYPE (GstAmDevice, gst_am_device, GST_TYPE_DEVICE); + +GST_DEVICE_PROVIDER_REGISTER_DEFINE (amdeviceprovider, "amdeviceprovider", + GST_RANK_PRIMARY, GST_TYPE_AM_DEVICE_PROVIDER); + +GST_DEBUG_CATEGORY_STATIC (gst_am_device_provider_debug); +#define GST_CAT_DEFAULT (gst_am_device_provider_debug) +#define device_provider_parent_class gst_am_device_provider_parent_class + +#define GET_DEVICES_ALL 0x00000003 + +/* from: https://developer.android.com/reference/android/media/AudioDeviceInfo */ +typedef enum +{ + TYPE_AUX_LINE = 19, + TYPE_BLE_BROADCAST = 30, + TYPE_BLE_HEADSET = 26, + TYPE_BLE_SPEAKER = 27, + TYPE_BLUETOOTH_A2DP = 8, + TYPE_BLUETOOTH_SCO = 7, + TYPE_BUILTIN_EARPIECE = 1, + TYPE_BUILTIN_MIC = 15, + TYPE_BUILTIN_SPEAKER = 2, + TYPE_BUILTIN_SPEAKER_SAFE = 24, + TYPE_BUS = 21, + TYPE_DOCK = 13, + TYPE_FM = 14, + TYPE_FM_TUNER = 16, + TYPE_HDMI = 9, + TYPE_HDMI_ARC = 10, + TYPE_HDMI_EARC = 29, + TYPE_HEARING_AID = 23, + TYPE_IP = 20, + TYPE_LINE_ANALOG = 5, + TYPE_LINE_DIGITAL = 6, + TYPE_REMOTE_SUBMIX = 25, + TYPE_TELEPHONY = 18, + TYPE_TV_TUNER = 17, + TYPE_UNKNOWN = 0, + TYPE_USB_ACCESSORY = 12, + TYPE_USB_DEVICE = 11, + TYPE_USB_HEADSET = 22, + TYPE_WIRED_HEADPHONES = 4, + TYPE_WIRED_HEADSET = 3, +} AudioInfo_DeviceType; + +static jobject +jni_get_global_context (JNIEnv * env) +{ + jclass activityThread; + jmethodID currentActivityThread; + jobject activity; + jmethodID getApplication; + jobject ctx; + + activityThread = (*env)->FindClass (env, "android/app/ActivityThread"); + currentActivityThread = + (*env)->GetStaticMethodID (env, activityThread, "currentActivityThread", + "()Landroid/app/ActivityThread;"); + activity = (*env)->CallStaticObjectMethod (env, activityThread, + currentActivityThread); + getApplication = + (*env)->GetMethodID (env, activityThread, "getApplication", + "()Landroid/app/Application;"); + + ctx = (*env)->CallObjectMethod (env, activity, getApplication); + return ctx; +} + +static jobject +jni_get_AudioManager_Instance (JNIEnv * env, jobject context) +{ + jclass contextClass; + jfieldID audioServiceField; + jstring audioService; + jmethodID getSystemServiceID; + jobject audioManager; + + contextClass = (*env)->FindClass (env, "android/content/Context"); + audioServiceField = + (*env)->GetStaticFieldID (env, contextClass, "AUDIO_SERVICE", + "Ljava/lang/String;"); + audioService = + (jstring) (*env)->GetStaticObjectField (env, contextClass, + audioServiceField); + getSystemServiceID = + (*env)->GetMethodID (env, contextClass, "getSystemService", + "(Ljava/lang/String;)Ljava/lang/Object;"); + + audioManager = (*env)->CallObjectMethod (env, context, + getSystemServiceID, audioService); + + g_assert (audioManager != NULL); + + return audioManager; +} + +static gint +jni_AudioInfo_getId (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + jmethodID getId; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + getId = (*env)->GetMethodID (env, audioInfoClass, "getId", "()I"); + + return (gint) (*env)->CallIntMethod (env, audioInfo, getId); +} + +static gboolean +jni_AudioInfo_isSink (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + jmethodID isSink; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + isSink = (*env)->GetMethodID (env, audioInfoClass, "isSink", "()Z"); + + return (gint) (*env)->CallBooleanMethod (env, audioInfo, isSink); +} + +static gboolean +jni_AudioInfo_isSource (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + jmethodID isSource; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + isSource = (*env)->GetMethodID (env, audioInfoClass, "isSource", "()Z"); + + return (gint) (*env)->CallBooleanMethod (env, audioInfo, isSource); +} + +static gchar * +jni_String_GetStringUTFChars (JNIEnv * env, jstring str) +{ + gchar *value = NULL; + const char *utf; + + utf = (*env)->GetStringUTFChars (env, str, NULL); + if (utf) { + value = g_strdup (utf); + (*env)->ReleaseStringUTFChars (env, str, utf); + } + + return value; +} + +static jstring +jni_Object_toString (JNIEnv * env, jobject obj) +{ + jclass objClass; + jmethodID toString; + + objClass = (*env)->FindClass (env, "java/lang/Object"); + toString = + (*env)->GetMethodID (env, objClass, "toString", "()Ljava/lang/String;"); + + return (*env)->CallObjectMethod (env, obj, toString); +} + +static gchar * +jni_AudioInfo_getProductName (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + + jmethodID getProductName; + + jobject productNameCharSeq; + jobject productNameString; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + + getProductName = + (*env)->GetMethodID (env, audioInfoClass, "getProductName", + "()Ljava/lang/CharSequence;"); + + productNameCharSeq = + (*env)->CallObjectMethod (env, audioInfo, getProductName); + productNameString = jni_Object_toString (env, productNameCharSeq); + + return jni_String_GetStringUTFChars (env, productNameString); +} + +static void +gst_am_device_provider_remove_device (GstAmDeviceProvider * amprovider, + JNIEnv * env, jobject audioInfo) +{ + GstDeviceProvider *provider = GST_DEVICE_PROVIDER_CAST (amprovider); + GstDevice *device = NULL; + GList *item; + gint id_to_remove; + + id_to_remove = jni_AudioInfo_getId (env, audioInfo); + + GST_OBJECT_LOCK (provider); + for (item = provider->devices; item; item = item->next) { + device = item->data; + gint device_id; + + g_object_get (device, "device-id", &device_id, NULL); + if (id_to_remove == device_id) { + gst_object_ref (device); + break; + } + + device = NULL; + } + GST_OBJECT_UNLOCK (provider); + + if (device) { + GST_DEBUG_OBJECT (provider, "Removing %" GST_PTR_FORMAT, device); + gst_device_provider_device_remove (provider, GST_DEVICE (device)); + g_object_unref (device); + } +} + +static gchar * +jni_get_AudioInfo_DeviceType_Name (AudioInfo_DeviceType deviceType) +{ + switch (deviceType) { + case TYPE_BLE_BROADCAST: + case TYPE_BLE_HEADSET: + case TYPE_BLE_SPEAKER: + case TYPE_BLUETOOTH_A2DP: + case TYPE_BLUETOOTH_SCO: + return g_strdup ("Bluetooth"); + + case TYPE_BUILTIN_EARPIECE: + case TYPE_BUILTIN_MIC: + case TYPE_BUILTIN_SPEAKER: + case TYPE_BUILTIN_SPEAKER_SAFE: + return g_strdup ("Built-in"); + + case TYPE_HDMI: + case TYPE_HDMI_ARC: + case TYPE_HDMI_EARC: + return g_strdup ("HDMI"); + + case TYPE_USB_ACCESSORY: + case TYPE_USB_DEVICE: + case TYPE_USB_HEADSET: + return g_strdup ("USB"); + + case TYPE_WIRED_HEADPHONES: + case TYPE_WIRED_HEADSET: + return g_strdup ("Wired"); + + default: + break; + } + + return NULL; +} + +static AudioInfo_DeviceType +jni_AudioInfo_getDeviceType (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + jmethodID getType; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + getType = (*env)->GetMethodID (env, audioInfoClass, "getType", "()I"); + + return (AudioInfo_DeviceType) (*env)->CallIntMethod (env, audioInfo, getType); +} + + +static GstDevice * +gst_am_device_from_audio_info (JNIEnv * env, jobject audioInfo) +{ + gint id; + gchar *product_name; + gchar *display_name; + gchar *type_name; + gboolean is_sink; + gboolean is_source; + AudioInfo_DeviceType deviceType; + GstDevice *device; + GstCaps *caps; + const gchar *device_class = NULL; + + is_sink = jni_AudioInfo_isSink (env, audioInfo); + is_source = jni_AudioInfo_isSource (env, audioInfo); + + if (!is_sink && !is_source) + return NULL; + + deviceType = jni_AudioInfo_getDeviceType (env, audioInfo); + type_name = jni_get_AudioInfo_DeviceType_Name (deviceType); + + /* filter the enumerated devices by supported type */ + if (!type_name) + return NULL; + + id = jni_AudioInfo_getId (env, audioInfo); + product_name = jni_AudioInfo_getProductName (env, audioInfo); + display_name = g_strdup_printf ("%s (%s)", product_name, type_name); + // FIXME: call getAudioProfiles() on the audioInfo and transform to caps + caps = gst_caps_new_empty_simple ("audio/x-raw"); + + g_free (type_name); + g_free (product_name); + + if (is_source) + device_class = "Audio/Source"; + if (is_sink) + device_class = "Audio/Sink"; + + device = g_object_new (GST_TYPE_AM_DEVICE, // + "device-id", id, // + "device-class", device_class, // + "display-name", display_name, // + "caps", caps, // + NULL); + + return GST_DEVICE_CAST (device); +} + +static void +gst_am_device_provider_add_device (GstAmDeviceProvider * provider, JNIEnv * env, + jobject audioInfo) +{ + GstDevice *device = gst_am_device_from_audio_info (env, audioInfo); + + if (device) { + GST_DEBUG_OBJECT (provider, "Adding %" GST_PTR_FORMAT, device); + gst_device_provider_device_add (GST_DEVICE_PROVIDER_CAST (provider), + device); + } +} + +static void +gst_am_device_provider_on_audio_devices_changed (GstAmDeviceProvider * provider, + JNIEnv * env, jobjectArray audioDevices, int action_type) +{ + jsize i; + jsize numAudioDevices; + + numAudioDevices = (*env)->GetArrayLength (env, audioDevices); + GST_DEBUG_OBJECT (provider, "Audio devices %s (%d)", + action_type == ADDED_DEVICES ? "added" : "removed", numAudioDevices); + + for (i = 0; i < numAudioDevices; i++) { + jobject audioInfo; + audioInfo = (*env)->GetObjectArrayElement (env, audioDevices, i); + + if (action_type == ADDED_DEVICES) + gst_am_device_provider_add_device (provider, env, audioInfo); + else if (action_type == REMOVED_DEVICES) + gst_am_device_provider_remove_device (provider, env, audioInfo); + } +} + +static void +gst_am_device_provider_on_audio_devices_added (JNIEnv * env, jobject thiz, + long long context, jobjectArray addedDevices) +{ + GstAmDeviceProvider *provider = + GST_AM_DEVICE_PROVIDER_CAST (JLONG_TO_GPOINTER (context)); + + gst_am_device_provider_on_audio_devices_changed (provider, env, + addedDevices, ADDED_DEVICES); +} + +static void +gst_am_device_provider_on_audio_devices_removed (JNIEnv * env, + jobject thiz, long long context, jobjectArray removedDevices) +{ + GstAmDeviceProvider *provider = + GST_AM_DEVICE_PROVIDER_CAST (JLONG_TO_GPOINTER (context)); + + gst_am_device_provider_on_audio_devices_changed (provider, env, + removedDevices, REMOVED_DEVICES); +} + +static jobject +gst_am_device_provider_create_callback (GstAmDeviceProvider * provider, + JNIEnv * env) +{ + jclass class; + jobject callback = NULL; + + jmethodID constructor_id; + jmethodID setContext; + + class = + gst_amc_jni_get_application_class (env, + "org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback", NULL); + if (!class) { + GST_ERROR ("Failed to load GstAmAudioDeviceCallback class"); + return NULL; + } + + constructor_id = + gst_amc_jni_get_method_id (env, NULL, class, "", "()V"); + if (!constructor_id) { + GST_ERROR ("Failed to get method id for constructor"); + goto done; + } + + setContext = + gst_amc_jni_get_method_id (env, NULL, class, "setContext", "(J)V"); + if (!setContext) { + GST_ERROR ("Can't find setContext method"); + goto done; + } + + callback = gst_amc_jni_new_object (env, NULL, TRUE, class, constructor_id); + + /* call setContext on GstAmAudioDeviceCallback */ + if (!gst_amc_jni_call_void_method (env, NULL, callback, setContext, + GPOINTER_TO_JLONG (provider))) { + GST_ERROR ("setContext call failed"); + goto done; + } + +done: + if (class) + gst_amc_jni_object_unref (env, class); + + return callback; +} + +static void +jni_AudioManager_registerAudioDeviceCallback (JNIEnv * env, + jobject audioManager, jobject audioDeviceCallback) +{ + jclass audioManagerClass; + jmethodID registerAudioDeviceCallback; + + audioManagerClass = (*env)->FindClass (env, "android/media/AudioManager"); + registerAudioDeviceCallback = + (*env)->GetMethodID (env, audioManagerClass, + "registerAudioDeviceCallback", + "(Landroid/media/AudioDeviceCallback;Landroid/os/Handler;)V"); + + (*env)->CallVoidMethod (env, audioManager, registerAudioDeviceCallback, + audioDeviceCallback, NULL); +} + +static void +jni_AudioManager_unregisterAudioDeviceCallback (JNIEnv * env, + jobject audioManager, jobject audioDeviceCallback) +{ + jclass audioManagerClass; + jmethodID unregisterAudioDeviceCallback; + + audioManagerClass = (*env)->FindClass (env, "android/media/AudioManager"); + unregisterAudioDeviceCallback = + (*env)->GetMethodID (env, audioManagerClass, + "unregisterAudioDeviceCallback", + "(Landroid/media/AudioDeviceCallback;)V"); + + (*env)->CallVoidMethod (env, audioManager, unregisterAudioDeviceCallback, + audioDeviceCallback); +} + +static jobjectArray +jni_AudioManager_getDevices (JNIEnv * env, jobject audioManager, + gint deviceType) +{ + jobjectArray audioDevices; // AudioDeviceInfo[] + jclass audioManagerClass; + jmethodID getDevicesID; + + audioManagerClass = (*env)->FindClass (env, "android/media/AudioManager"); + getDevicesID = (*env)->GetMethodID (env, audioManagerClass, "getDevices", + "(I)[Landroid/media/AudioDeviceInfo;"); + audioDevices = + (*env)->CallObjectMethod (env, audioManager, getDevicesID, deviceType); + + return audioDevices; +} + +static GList * +gst_am_device_provider_probe (GstDeviceProvider * object) +{ + GList *devices = NULL; + JNIEnv *env; + jobject ctx; + jobject audioManager; + jsize i, numAudioDevices; + jobjectArray audioDevices; + + env = gst_amc_jni_get_env (); + ctx = jni_get_global_context (env); + if (!ctx) { + GST_ERROR ("Can't get an instance of global context"); + return NULL; + } + + audioManager = jni_get_AudioManager_Instance (env, ctx); + if (!audioManager) { + GST_ERROR ("Can't get an instance of AudioManager"); + return NULL; + } + + audioDevices = + jni_AudioManager_getDevices (env, audioManager, GET_DEVICES_ALL); + if (!audioDevices) { + GST_ERROR ("No audio devices found!"); + return NULL; + } + + numAudioDevices = (*env)->GetArrayLength (env, audioDevices); + GST_DEBUG ("numAudioDevices: %u", numAudioDevices); + for (i = 0; i < numAudioDevices; i++) { + jobject audioInfo; + GstDevice *device; + + audioInfo = (*env)->GetObjectArrayElement (env, audioDevices, i); + device = gst_am_device_from_audio_info (env, audioInfo); + + if (device) { + gst_object_ref_sink (device); + devices = g_list_append (devices, device); + } + } + + return devices; +} + +static gboolean +gst_am_device_provider_start (GstDeviceProvider * object) +{ + GstAmDeviceProvider *provider; + JNIEnv *env; + jobject ctx; + jobject audioDeviceCallback; + jobject audioManager; + + provider = GST_AM_DEVICE_PROVIDER_CAST (object); + if (provider->audioDeviceCB != NULL) { + GST_ERROR ("Already started!"); + return FALSE; + } + + env = gst_amc_jni_get_env (); + audioDeviceCallback = gst_am_device_provider_create_callback (provider, env); + if (!audioDeviceCallback) { + return FALSE; + } + + GST_DEBUG_OBJECT (provider, "Starting..."); + + ctx = jni_get_global_context (env); + if (!ctx) { + GST_ERROR ("Can't get an instance of global context"); + return FALSE; + } + + audioManager = jni_get_AudioManager_Instance (env, ctx); + if (!audioManager) { + GST_ERROR ("Can't get an instance of AudioManager"); + return FALSE; + } + + provider->audioDeviceCB = audioDeviceCallback; + jni_AudioManager_registerAudioDeviceCallback (env, audioManager, + audioDeviceCallback); + + return TRUE; +} + +static void +gst_am_device_provider_stop (GstDeviceProvider * object) +{ + GstAmDeviceProvider *provider; + JNIEnv *env; + jobject ctx; + jobject audioManager; + + provider = GST_AM_DEVICE_PROVIDER_CAST (object); + if (provider->audioDeviceCB == NULL) { + GST_ERROR ("No callback set"); + return; + } + + GST_DEBUG_OBJECT (provider, "Stopping..."); + + env = gst_amc_jni_get_env (); + ctx = jni_get_global_context (env); + if (!ctx) { + GST_ERROR ("Can't get an instance of global context"); + return; + } + + audioManager = jni_get_AudioManager_Instance (env, ctx); + if (!audioManager) { + GST_ERROR ("Can't get an instance of AudioManager"); + return; + } + + jni_AudioManager_unregisterAudioDeviceCallback (env, audioManager, + provider->audioDeviceCB); + gst_amc_jni_object_unref (env, provider->audioDeviceCB); + provider->audioDeviceCB = NULL; +} + + +static void +gst_am_device_provider_populate_devices (GstDeviceProvider * provider) +{ + GList *devices, *it; + + devices = gst_am_device_provider_probe (provider); + + for (it = devices; it != NULL; it = g_list_next (it)) { + GstDevice *device = GST_DEVICE_CAST (it->data); + gst_device_provider_device_add (provider, device); + } + + g_list_free_full (devices, gst_object_unref); +} + +static void +gst_am_device_provider_init (GstAmDeviceProvider * provider) +{ + gst_am_device_provider_populate_devices (GST_DEVICE_PROVIDER_CAST (provider)); +} + +static void +gst_am_device_provider_finalize (GObject * object) +{ + GstAmDeviceProvider *provider = GST_AM_DEVICE_PROVIDER_CAST (object); + (void) provider; + + G_OBJECT_CLASS (device_provider_parent_class)->finalize (object); +} + +static void +gst_am_device_provider_register_jni_native_methods () +{ + jclass class; + JNIEnv *env; + + JNINativeMethod native_methods[] = { + {"native_onAudioDevicesAdded", "(J[Landroid/media/AudioDeviceInfo;)V", + (void *) gst_am_device_provider_on_audio_devices_added}, + {"native_onAudioDevicesRemoved", "(J[Landroid/media/AudioDeviceInfo;)V", + (void *) gst_am_device_provider_on_audio_devices_removed}, + }; + + env = gst_amc_jni_get_env (); + class = + gst_amc_jni_get_application_class (env, + "org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback", NULL); + if (!class) { + GST_ERROR ("Failed to load GstAmAudioDeviceCallback class"); + return; + } + + (*env)->RegisterNatives (env, class, + (const JNINativeMethod *) &native_methods, G_N_ELEMENTS (native_methods)); + if ((*env)->ExceptionCheck (env)) { + GST_ERROR ("Failed to register native methods"); + } + + gst_amc_jni_object_unref (env, class); +} + +static void +gst_am_device_provider_class_init (GstAmDeviceProviderClass * klass) +{ + GstDeviceProviderClass *dm_class = GST_DEVICE_PROVIDER_CLASS (klass); + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + dm_class->probe = gst_am_device_provider_probe; + dm_class->start = gst_am_device_provider_start; + dm_class->stop = gst_am_device_provider_stop; + + gobject_class->finalize = gst_am_device_provider_finalize; + + gst_device_provider_class_set_static_metadata (dm_class, + "Android Audio Device Provider", "Sink/Source/Audio", + "List and monitor Android audio devices", + "Tulio Beloqui "); + + GST_DEBUG_CATEGORY_INIT (gst_am_device_provider_debug, + "amdeviceprovider", 0, "Android Audio Device Provider"); + + gst_am_device_provider_register_jni_native_methods (); +} + +enum +{ + PROP_DEVICE_ID = 1, +}; + +static void +gst_am_device_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstAmDevice *device = GST_AM_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_DEVICE_ID: + g_value_set_int (value, device->device_id); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_am_device_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstAmDevice *device = GST_AM_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_DEVICE_ID: + device->device_id = g_value_get_int (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + + +static void +gst_am_device_class_init (GstAmDeviceClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->get_property = gst_am_device_get_property; + object_class->set_property = gst_am_device_set_property; + + g_object_class_install_property (object_class, PROP_DEVICE_ID, + g_param_spec_int ("device-id", "Audio Device ID", + "The audio device id", 0, G_MAXINT, -1, + G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); +} + +static void +gst_am_device_init (GstAmDevice * device) +{ +} diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.h b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.h new file mode 100644 index 00000000000..9b43c608c1d --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.h @@ -0,0 +1,44 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_AM_DEVICE_PROVIDER_H__ +#define __GST_AM_DEVICE_PROVIDER_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_AM_DEVICE_PROVIDER (gst_am_device_provider_get_type ()) +#define GST_AM_DEVICE_PROVIDER_CAST(obj) ((GstAmDeviceProvider *)(obj)) + +#define GST_TYPE_AM_DEVICE (gst_am_device_get_type ()) +#define GST_AM_DEVICE_CAST(obj) ((GstAmDevice *)(obj)) + +G_DECLARE_FINAL_TYPE (GstAmDevice, gst_am_device, GST, AM_DEVICE, + GstDevice) + +G_DECLARE_FINAL_TYPE (GstAmDeviceProvider, gst_am_device_provider, GST, + AM_DEVICE_PROVIDER, GstDeviceProvider) + +GST_DEVICE_PROVIDER_REGISTER_DECLARE (amdeviceprovider); + +G_END_DECLS + +#endif /* __GST_AM_DEVICE_PROVIDER_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/meson.build b/subprojects/gst-plugins-bad/sys/androidmedia/meson.build index 810f8c2feba..de00294709c 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/meson.build +++ b/subprojects/gst-plugins-bad/sys/androidmedia/meson.build @@ -74,6 +74,15 @@ else plugin_name = 'gstandroidmedia' endif +if have_jni_h + androidmedia_java_sources += [ + 'org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java', + ] + androidmedia_sources += [ + 'gstamdeviceprovider.c', + ] +endif + have_ndkcamera = cc.has_header('camera/NdkCameraError.h', required : false) if have_ndkcamera androidmedia_sources += [ diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java b/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java new file mode 100644 index 00000000000..d88cff80d78 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java @@ -0,0 +1,45 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +package org.freedesktop.gstreamer.androidmedia; + +import android.media.AudioDeviceInfo; +import android.media.AudioDeviceCallback; + +public class GstAmAudioDeviceCallback extends AudioDeviceCallback +{ + private long context = 0; + + public void setContext(long c) { + context = c; + } + + public synchronized void onAudioDevicesAdded(AudioDeviceInfo[] addedDevices) { + native_onAudioDevicesAdded(context, addedDevices); + } + + public synchronized void onAudioDevicesRemoved(AudioDeviceInfo[] removedDevices) { + native_onAudioDevicesRemoved(context, removedDevices); + } + + private native void native_onAudioDevicesAdded (long context, AudioDeviceInfo[] addedDevices); + + private native void native_onAudioDevicesRemoved (long context, AudioDeviceInfo[] addedDevices); +} From 5e519048485d77531137c5e134d81cce690a9cd9 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 29 Jun 2022 12:58:05 +0200 Subject: [PATCH 240/339] gstamdeviceprovider: fixed a race condition in which the callback would execute regardless of the provider being stopped. --- .../sys/androidmedia/gstamdeviceprovider.c | 87 +++++++++++-------- .../GstAmAudioDeviceCallback.java | 6 +- 2 files changed, 57 insertions(+), 36 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c index 0d9f2b44cdd..ad768b006e8 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c @@ -422,14 +422,12 @@ gst_am_device_provider_on_audio_devices_removed (JNIEnv * env, } static jobject -gst_am_device_provider_create_callback (GstAmDeviceProvider * provider, - JNIEnv * env) +gst_am_device_provider_create_callback (JNIEnv * env) { jclass class; jobject callback = NULL; jmethodID constructor_id; - jmethodID setContext; class = gst_amc_jni_get_application_class (env, @@ -446,27 +444,41 @@ gst_am_device_provider_create_callback (GstAmDeviceProvider * provider, goto done; } - setContext = - gst_amc_jni_get_method_id (env, NULL, class, "setContext", "(J)V"); - if (!setContext) { - GST_ERROR ("Can't find setContext method"); - goto done; + callback = gst_amc_jni_new_object (env, NULL, TRUE, class, constructor_id); + +done: + if (class) + gst_amc_jni_object_unref (env, class); + + return callback; +} + +static void +gst_am_device_provider_callback_set_context (JNIEnv * env, + jobject audioDeviceCallback, jlong context) +{ + jclass class; + jmethodID setContext; + + class = + gst_amc_jni_get_application_class (env, + "org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback", NULL); + if (!class) { + GST_ERROR ("Failed to load GstAmAudioDeviceCallback class"); + return; } - callback = gst_amc_jni_new_object (env, NULL, TRUE, class, constructor_id); + setContext = + gst_amc_jni_get_method_id (env, NULL, class, "setContext", "(J)V"); /* call setContext on GstAmAudioDeviceCallback */ - if (!gst_amc_jni_call_void_method (env, NULL, callback, setContext, - GPOINTER_TO_JLONG (provider))) { + if (!gst_amc_jni_call_void_method (env, NULL, audioDeviceCallback, setContext, + context)) { GST_ERROR ("setContext call failed"); - goto done; } -done: if (class) gst_amc_jni_object_unref (env, class); - - return callback; } static void @@ -568,6 +580,22 @@ gst_am_device_provider_probe (GstDeviceProvider * object) return devices; } +static void +gst_am_device_provider_populate_devices (GstDeviceProvider * provider) +{ + GList *devices, *it; + + devices = gst_am_device_provider_probe (provider); + + for (it = devices; it != NULL; it = g_list_next (it)) { + GstDevice *device = GST_DEVICE_CAST (it->data); + if (device) + gst_device_provider_device_add (provider, device); + } + + g_list_free_full (devices, gst_object_unref); +} + static gboolean gst_am_device_provider_start (GstDeviceProvider * object) { @@ -583,13 +611,19 @@ gst_am_device_provider_start (GstDeviceProvider * object) return FALSE; } + GST_DEBUG_OBJECT (provider, "Starting..."); + + gst_am_device_provider_populate_devices (object); + env = gst_amc_jni_get_env (); - audioDeviceCallback = gst_am_device_provider_create_callback (provider, env); + audioDeviceCallback = gst_am_device_provider_create_callback (env); if (!audioDeviceCallback) { return FALSE; } - GST_DEBUG_OBJECT (provider, "Starting..."); + gst_am_device_provider_callback_set_context (env, audioDeviceCallback, + GPOINTER_TO_JLONG (provider)); + ctx = jni_get_global_context (env); if (!ctx) { @@ -639,32 +673,17 @@ gst_am_device_provider_stop (GstDeviceProvider * object) return; } + gst_am_device_provider_callback_set_context (env, provider->audioDeviceCB, 0); + jni_AudioManager_unregisterAudioDeviceCallback (env, audioManager, provider->audioDeviceCB); gst_amc_jni_object_unref (env, provider->audioDeviceCB); provider->audioDeviceCB = NULL; } - -static void -gst_am_device_provider_populate_devices (GstDeviceProvider * provider) -{ - GList *devices, *it; - - devices = gst_am_device_provider_probe (provider); - - for (it = devices; it != NULL; it = g_list_next (it)) { - GstDevice *device = GST_DEVICE_CAST (it->data); - gst_device_provider_device_add (provider, device); - } - - g_list_free_full (devices, gst_object_unref); -} - static void gst_am_device_provider_init (GstAmDeviceProvider * provider) { - gst_am_device_provider_populate_devices (GST_DEVICE_PROVIDER_CAST (provider)); } static void diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java b/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java index d88cff80d78..cb1f1f9e5e8 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java +++ b/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java @@ -32,11 +32,13 @@ public void setContext(long c) { } public synchronized void onAudioDevicesAdded(AudioDeviceInfo[] addedDevices) { - native_onAudioDevicesAdded(context, addedDevices); + if (context != 0) + native_onAudioDevicesAdded(context, addedDevices); } public synchronized void onAudioDevicesRemoved(AudioDeviceInfo[] removedDevices) { - native_onAudioDevicesRemoved(context, removedDevices); + if (context != 0) + native_onAudioDevicesRemoved(context, removedDevices); } private native void native_onAudioDevicesAdded (long context, AudioDeviceInfo[] addedDevices); From 76e1b8bd2de68bd3db535f3364b4b39370a67526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Sun, 3 Jul 2022 14:10:43 +0200 Subject: [PATCH 241/339] applemedia/vtenc: fix deadlock Calling destroy_session() without first calling finish() deadlocks almost every time, due to the explanation given in gst_vtenc_finish_encoding() --- subprojects/gst-plugins-bad/sys/applemedia/vtenc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c index 4c2fb147b5f..cbcba3e5b4d 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c @@ -1004,6 +1004,8 @@ gst_vtenc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state) gst_video_codec_state_unref (self->input_state); } + gst_vtenc_finish (enc); + GST_OBJECT_LOCK (self); gst_vtenc_destroy_session (self, &self->session); GST_OBJECT_UNLOCK (self); From 79d1c1185c4911625a8b2f9a888d3a3ebcfbb70c Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 26 Jul 2022 13:47:01 +0200 Subject: [PATCH 242/339] applemedia/avaudiodeviceprovider: initial commit --- .../sys/applemedia/avaudiodeviceprovider.h | 49 ++++ .../sys/applemedia/avaudiodeviceprovider.m | 241 ++++++++++++++++++ .../sys/applemedia/meson.build | 3 +- .../gst-plugins-bad/sys/applemedia/plugin.m | 11 +- 4 files changed, 300 insertions(+), 4 deletions(-) create mode 100644 subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.h create mode 100644 subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.m diff --git a/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.h b/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.h new file mode 100644 index 00000000000..b4cd0b2c2ff --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.h @@ -0,0 +1,49 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_AV_AUDIO_DEVICE_PROVIDER_H__ +#define __GST_AV_AUDIO_DEVICE_PROVIDER_H__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_AV_AUDIO_DEVICE_PROVIDER (gst_av_audio_device_provider_get_type ()) +#define GST_AV_AUDIO_DEVICE_PROVIDER_CAST(obj) ((GstAVAudioDeviceProvider *)(obj)) + +#define GST_TYPE_AV_AUDIO_DEVICE (gst_av_audio_device_get_type ()) +#define GST_AV_AUDIO_DEVICE_CAST(obj) ((GstAVAudioDevice *)(obj)) + +G_DECLARE_FINAL_TYPE (GstAVAudioDevice, gst_av_audio_device, GST, AV_AUDIO_DEVICE, + GstDevice) + +G_DECLARE_FINAL_TYPE (GstAVAudioDeviceProvider, gst_av_audio_device_provider, GST, + AV_AUDIO_DEVICE_PROVIDER, GstDeviceProvider) + +GST_DEVICE_PROVIDER_REGISTER_DECLARE (avaudiodeviceprovider); + +G_END_DECLS + + +#endif /* __GST_AV_AUDIO_DEVICE_PROVIDER_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.m b/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.m new file mode 100644 index 00000000000..5e15631a779 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.m @@ -0,0 +1,241 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#import + +#include "avaudiodeviceprovider.h" + +struct _GstAVAudioDeviceProvider +{ + GstDeviceProvider parent; +}; + +struct _GstAVAudioDevice +{ + GstDevice parent; + + gchar *type; + gchar *uid; +}; + +G_DEFINE_TYPE (GstAVAudioDeviceProvider, gst_av_audio_device_provider, + GST_TYPE_DEVICE_PROVIDER); + +G_DEFINE_TYPE (GstAVAudioDevice, gst_av_audio_device, GST_TYPE_DEVICE); + +GST_DEVICE_PROVIDER_REGISTER_DEFINE (avaudiodeviceprovider, + "avaudiodeviceprovider", GST_RANK_PRIMARY, + GST_TYPE_AV_AUDIO_DEVICE_PROVIDER); + +GST_DEBUG_CATEGORY_STATIC (gst_av_audio_device_provider_debug); +#define GST_CAT_DEFAULT (gst_av_audio_device_provider_debug) + +static GstDevice * +gst_av_audio_device_new (gchar * name, gchar * type, gchar * uid, + const gchar * device_class) +{ + GstAVAudioDevice *device = g_object_new (GST_TYPE_AV_AUDIO_DEVICE, + "display-name", name, + "device-class", device_class, + "type", type, + "uid", uid, + "caps", gst_caps_new_empty_simple ("audio/x-raw"), + NULL); + + return GST_DEVICE_CAST (device); +} + +static GstDevice * +gst_device_from_port_description (AVAudioSessionPortDescription * + portDesc, const gchar * device_class) +{ + gchar *name = g_strdup ([portDesc.portName UTF8String]); + gchar *type = g_strdup ([portDesc.portType UTF8String]); + gchar *uid = g_strdup ([portDesc.UID UTF8String]); + + return gst_av_audio_device_new (name, type, uid, device_class); +} + +static GList * +gst_av_audio_device_provider_probe (GstDeviceProvider * object) +{ + GList *devices = NULL; + AVAudioSession *audioSession =[AVAudioSession sharedInstance]; + + NSString *sessionCategory = AVAudioSessionCategoryPlayAndRecord; + AVAudioSessionCategoryOptions options = + AVAudioSessionCategoryOptionAllowBluetooth | + AVAudioSessionCategoryOptionAllowBluetoothA2DP | + AVAudioSessionCategoryOptionDefaultToSpeaker; + +[audioSession setCategory: sessionCategory withOptions: options error:nil]; +[audioSession setMode: AVAudioSessionModeVoiceChat error:nil]; +[audioSession setActive: YES error:nil]; + + AVAudioSessionRouteDescription *route = audioSession.currentRoute; + + gboolean addedBuiltinSpk = FALSE; + gboolean addedBuiltinRecv = FALSE; + + if (route != nil) { + for (AVAudioSessionPortDescription * outPort in route.outputs) { + if ([outPort.portType isEqualToString:AVAudioSessionPortBuiltInSpeaker]) + addedBuiltinSpk = TRUE; + + if ([outPort.portType isEqualToString:AVAudioSessionPortBuiltInReceiver]) + addedBuiltinRecv = TRUE; + + GstDevice *device = + gst_device_from_port_description (outPort, "Audio/Sink"); + devices = g_list_append (devices, gst_object_ref_sink (device)); + } + } + + for (AVAudioSessionPortDescription * inPort in[audioSession availableInputs]) { + GstDevice *device = + gst_device_from_port_description (inPort, "Audio/Source"); + devices = g_list_append (devices, gst_object_ref_sink (device)); + } + + if (!addedBuiltinSpk) { + gchar *type = g_strdup ([AVAudioSessionPortBuiltInSpeaker UTF8String]); + gchar *name = g_strdup_printf ("iPhone %s", type); + GstDevice *dev = gst_av_audio_device_new (name, type, NULL, "Audio/Sink"); + + devices = g_list_append (devices, gst_object_ref_sink (dev)); + } + + if (!addedBuiltinRecv) { + gchar *type = g_strdup ([AVAudioSessionPortBuiltInReceiver UTF8String]); + gchar *name = g_strdup_printf ("iPhone %s", type); + GstDevice *dev = gst_av_audio_device_new (name, type, NULL, "Audio/Sink"); + + devices = g_list_append (devices, gst_object_ref_sink (dev)); + } + + return devices; +} + +static void +gst_av_audio_device_provider_init (GstAVAudioDeviceProvider * provider) +{ +} + +static void +gst_av_audio_device_provider_class_init (GstAVAudioDeviceProviderClass * klass) +{ + GstDeviceProviderClass *dm_class = GST_DEVICE_PROVIDER_CLASS (klass); + dm_class->probe = gst_av_audio_device_provider_probe; + + gst_device_provider_class_set_static_metadata (dm_class, + "AV Audio Device Provider", "Sink/Source/Audio", + "List and monitor IOS audio devices", "Tulio Beloqui "); + + GST_DEBUG_CATEGORY_INIT (gst_av_audio_device_provider_debug, + "avaudiodeviceprovider", 0, "AV Audio Device Provider for IOS"); +} + +enum +{ + PROP_DEVICE_TYPE = 1, + PROP_DEVICE_UID, +}; + +static void +gst_av_audio_device_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstAVAudioDevice *device = GST_AV_AUDIO_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_DEVICE_TYPE: + g_value_set_string (value, device->type); + break; + case PROP_DEVICE_UID: + g_value_set_string (value, device->uid); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_av_audio_device_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstAVAudioDevice *device = GST_AV_AUDIO_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_DEVICE_TYPE: + device->type = g_strdup (g_value_get_string (value)); + break; + case PROP_DEVICE_UID: + device->uid = g_strdup (g_value_get_string (value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_av_audio_device_finalize (GObject * object) +{ + GstAVAudioDevice *device = GST_AV_AUDIO_DEVICE_CAST (object); + + g_free (device->type); + g_free (device->uid); + + G_OBJECT_CLASS (gst_av_audio_device_parent_class)->finalize (object); +} + +static void +gst_av_audio_device_class_init (GstAVAudioDeviceClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->get_property = gst_av_audio_device_get_property; + object_class->set_property = gst_av_audio_device_set_property; + object_class->finalize = gst_av_audio_device_finalize; + + g_object_class_install_property (object_class, PROP_DEVICE_TYPE, + g_param_spec_string ("type", + "Type", + "The AVAudioSessionPort type for the device", + NULL, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (object_class, PROP_DEVICE_UID, + g_param_spec_string ("uid", + "UID", + "A system-assigned unique identifier (UID) for the device", + NULL, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_av_audio_device_init (GstAVAudioDevice * device) +{ +} diff --git a/subprojects/gst-plugins-bad/sys/applemedia/meson.build b/subprojects/gst-plugins-bad/sys/applemedia/meson.build index 5c285cec5ab..52f996c59a7 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/meson.build +++ b/subprojects/gst-plugins-bad/sys/applemedia/meson.build @@ -68,7 +68,8 @@ endif if host_system == 'ios' applemedia_sources += [ 'iosassetsrc.m', - 'iosglmemory.c' + 'iosglmemory.c', + 'avaudiodeviceprovider.m', ] applemedia_objc_args += ['-fobjc-abi-version=2', '-fobjc-legacy-dispatch'] diff --git a/subprojects/gst-plugins-bad/sys/applemedia/plugin.m b/subprojects/gst-plugins-bad/sys/applemedia/plugin.m index 9fc42d728a0..ec2b9910539 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/plugin.m +++ b/subprojects/gst-plugins-bad/sys/applemedia/plugin.m @@ -26,6 +26,7 @@ #ifdef HAVE_IOS #include "iosassetsrc.h" #include "iosglmemory.h" +#include "avaudiodeviceprovider.h" #endif #ifdef HAVE_AVFOUNDATION #include "avfvideosrc.h" @@ -52,7 +53,7 @@ static void enable_mt_mode (void) { - NSThread * th = [[NSThread alloc] init]; + NSThread *th =[[NSThread alloc] init]; [th start]; g_assert ([NSThread isMultiThreaded]); } @@ -70,6 +71,9 @@ res &= gst_element_register (plugin, "iosassetsrc", GST_RANK_SECONDARY, GST_TYPE_IOS_ASSET_SRC); + res &= + gst_device_provider_register (plugin, "avaudiodeviceprovider", GST_RANK_PRIMARY, + GST_TYPE_AV_AUDIO_DEVICE_PROVIDER); #else enable_mt_mode (); #endif @@ -82,10 +86,11 @@ res &= gst_element_register (plugin, "avsamplebufferlayersink", GST_RANK_NONE, GST_TYPE_AV_SAMPLE_VIDEO_SINK); res &= gst_device_provider_register (plugin, "avfdeviceprovider", - GST_RANK_PRIMARY, GST_TYPE_AVF_DEVICE_PROVIDER); + GST_RANK_PRIMARY, GST_TYPE_AVF_DEVICE_PROVIDER); #endif - res &= gst_element_register (plugin, "atdec", GST_RANK_MARGINAL, GST_TYPE_ATDEC); + res &= + gst_element_register (plugin, "atdec", GST_RANK_MARGINAL, GST_TYPE_ATDEC); #ifdef HAVE_VIDEOTOOLBOX /* Check if the framework actually exists at runtime */ From b6d3bd219ad6de6068c12b16034cdce5e44f06cb Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 9 Aug 2022 13:03:38 +0200 Subject: [PATCH 243/339] winks/ksvideohelper: do not throw a warning if the media type is not supported --- subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c | 1 - 1 file changed, 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c b/subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c index a5b53c15bee..fee908176e4 100644 --- a/subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c +++ b/subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c @@ -547,7 +547,6 @@ ks_video_probe_filter_for_caps (HANDLE filter_handle) range->Specifier, &entry->is_rgb); if (media_structure == NULL) { - g_warning ("ks_video_format_to_structure returned NULL"); ks_video_media_type_free (entry); entry = NULL; } else if (src_vscc == NULL) { From 6b99618a9158a8deaf2cf1e73d93697c3778d0d6 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 22 Aug 2022 15:40:53 +0200 Subject: [PATCH 244/339] gstpad: gst-indent --- subprojects/gstreamer/gst/gstpad.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/subprojects/gstreamer/gst/gstpad.c b/subprojects/gstreamer/gst/gstpad.c index 2e661356349..515c28fcd80 100644 --- a/subprojects/gstreamer/gst/gstpad.c +++ b/subprojects/gstreamer/gst/gstpad.c @@ -226,11 +226,13 @@ static GstFlowQuarks flow_quarks[] = { }; static void -gst_pad_warn_if_unlinked (GstPad *pad, GstFlowReturn res) { +gst_pad_warn_if_unlinked (GstPad * pad, GstFlowReturn res) +{ if (G_UNLIKELY (GST_PAD_PEER (pad) == NULL && res == GST_FLOW_NOT_LINKED)) { if (!pad->priv->warned_unlinked) { pad->priv->warned_unlinked = TRUE; - GST_WARNING_OBJECT (pad, "%p:%p returning not-linked", GST_PAD_PARENT(pad), pad); + GST_WARNING_OBJECT (pad, "%p:%p returning not-linked", + GST_PAD_PARENT (pad), pad); } } else { pad->priv->warned_unlinked = FALSE; From f0a64132f6c6144d5aa7273c6f06a48b5c25a0d2 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 2 Sep 2022 13:21:36 +0200 Subject: [PATCH 245/339] gstpad: PEXHACK: log chain(list)funcs that take longer than 100ms Remove this before releasing please --- subprojects/gstreamer/gst/gstpad.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/subprojects/gstreamer/gst/gstpad.c b/subprojects/gstreamer/gst/gstpad.c index 515c28fcd80..93c2a810872 100644 --- a/subprojects/gstreamer/gst/gstpad.c +++ b/subprojects/gstreamer/gst/gstpad.c @@ -4429,6 +4429,10 @@ gst_pad_chain_data_unchecked (GstPad * pad, GstPadProbeType type, void *data) GST_TRACER_PAD_CHAIN_PRE (pad, data); } +/* FIXME: PEXHACK BEGIN */ + gint64 start, dur; +/* FIXME: PEXHACK END */ + GST_PAD_STREAM_LOCK (pad); GST_OBJECT_LOCK (pad); @@ -4480,8 +4484,20 @@ gst_pad_chain_data_unchecked (GstPad * pad, GstPadProbeType type, void *data) "calling chainfunction &%s with buffer %" GST_PTR_FORMAT, GST_DEBUG_FUNCPTR_NAME (chainfunc), GST_BUFFER (data)); +/* FIXME: PEXHACK BEGIN */ + start = g_get_monotonic_time (); +/* FIXME: PEXHACK END */ + ret = chainfunc (pad, parent, GST_BUFFER_CAST (data)); +/* FIXME: PEXHACK BEGIN */ + dur = (g_get_monotonic_time () - start) / G_TIME_SPAN_MILLISECOND; + if (dur >= 100) { + GST_WARNING_OBJECT (pad, "Slow chainfunc (%s) %" G_GINT64_FORMAT "ms", + GST_DEBUG_FUNCPTR_NAME (chainfunc), dur); + } +/* FIXME: PEXHACK END */ + GST_CAT_DEBUG_OBJECT (GST_CAT_SCHEDULING, pad, "called chainfunction &%s with buffer %p, returned %s", GST_DEBUG_FUNCPTR_NAME (chainfunc), data, gst_flow_get_name (ret)); @@ -4495,8 +4511,20 @@ gst_pad_chain_data_unchecked (GstPad * pad, GstPadProbeType type, void *data) "calling chainlistfunction &%s", GST_DEBUG_FUNCPTR_NAME (chainlistfunc)); +/* FIXME: PEXHACK BEGIN */ + start = g_get_monotonic_time (); +/* FIXME: PEXHACK END */ + ret = chainlistfunc (pad, parent, GST_BUFFER_LIST_CAST (data)); +/* FIXME: PEXHACK BEGIN */ + dur = (g_get_monotonic_time () - start) / G_TIME_SPAN_MILLISECOND; + if (dur >= 100) { + GST_WARNING_OBJECT (pad, "Slow chainlistfunc (%s) %" G_GINT64_FORMAT "ms", + GST_DEBUG_FUNCPTR_NAME (chainlistfunc), dur); + } +/* FIXME: PEXHACK END */ + GST_CAT_DEBUG_OBJECT (GST_CAT_SCHEDULING, pad, "called chainlistfunction &%s, returned %s", GST_DEBUG_FUNCPTR_NAME (chainlistfunc), gst_flow_get_name (ret)); From 9f5eb5edec2db3c541d023f7fd7606ad633e9ee0 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 26 Aug 2022 16:01:40 +0200 Subject: [PATCH 246/339] ahc2src: cleanups and fixes --- .../sys/androidmedia/gstahc2src.c | 102 ++++++++++++------ 1 file changed, 67 insertions(+), 35 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c index 8107f36bb92..9eca4b2f291 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c @@ -60,9 +60,10 @@ struct _GstAHC2Src GstDataQueue *outbound_queue; GstClockTime previous_ts; gboolean started; - GMutex mutex; gint max_images; + volatile gint referenced_images; + gint camera_index; ACameraDevice *camera_device; ACameraManager *camera_manager; @@ -84,7 +85,7 @@ struct _GstAHC2Src typedef struct { - gsize refcount; + volatile gint refcount; GstAHC2Src *ahc2src; AImage *image; } GstWrappedAImage; @@ -101,10 +102,9 @@ gst_wrapped_aimage_ref (GstWrappedAImage * self) g_return_val_if_fail (self->refcount >= 1, NULL); GST_TRACE_OBJECT (self->ahc2src, - "ref GstWrappedAImage %p, refcount: %" G_GSIZE_FORMAT, self, - self->refcount); + "ref GstWrappedAImage %p, refcount: %d", self, self->refcount); - self->refcount++; + g_atomic_int_add (&self->refcount, 1); return self; } @@ -117,11 +117,16 @@ gst_wrapped_aimage_unref (GstWrappedAImage * self) g_return_if_fail (self->refcount >= 1); GST_TRACE_OBJECT (self->ahc2src, - "unref GstWrappedAImage %p, refcount: %" G_GSIZE_FORMAT, self, - self->refcount); - if (--self->refcount == 0) { + "unref GstWrappedAImage %p, refcount: %d", self, self->refcount); + gint old_ref = g_atomic_int_add (&self->refcount, -1); + + if (old_ref == 1) { + g_atomic_int_add (&self->ahc2src->referenced_images, -1); + GST_TRACE_OBJECT (self->ahc2src, + "Destroying image:%p, referenced_images: %d", self->image, + self->ahc2src->referenced_images); + AImage_delete (self->image); gst_object_unref (self->ahc2src); - g_clear_pointer (&self->image, (GDestroyNotify) AImage_delete); } } @@ -213,7 +218,7 @@ gst_ahc2_src_get_capabilities (GstPhotography * photo) } } - g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + ACameraMetadata_free (metadata); return caps; } @@ -482,7 +487,7 @@ gst_ahc2_src_get_ev_compensation (GstPhotography * photo, gfloat * ev_comp) ret = TRUE; out: - g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + ACameraMetadata_free (metadata); return ret; } @@ -551,7 +556,7 @@ gst_ahc2_src_set_ev_compensation (GstPhotography * photo, gfloat ev_comp) ret = TRUE; out: - g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + ACameraMetadata_free (metadata); return ret; } @@ -1025,7 +1030,7 @@ gst_ahc2_src_set_zoom (GstPhotography * photo, gfloat zoom) ret = TRUE; out: - g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + ACameraMetadata_free (metadata); return ret; } @@ -1100,6 +1105,8 @@ gst_ahc2_src_set_caps (GstBaseSrc * src, GstCaps * caps) GST_WARNING_OBJECT (self, "framerate holds unrecognizable value"); } + GST_OBJECT_LOCK (self); + if (fps_min != 0 && fps_max != 0) { gint fps_range[2] = { fps_min, fps_max }; ACaptureRequest_setEntry_i32 (self->capture_request, @@ -1116,6 +1123,17 @@ gst_ahc2_src_set_caps (GstBaseSrc * src, GstCaps * caps) goto failed; } + /* we flush the internal queue, so that old AImages can be cleaned out */ + gst_data_queue_flush (self->outbound_queue); + + while (self->referenced_images > 0) { + g_usleep (G_USEC_PER_SEC / 50); + GST_DEBUG_OBJECT (self, + "waiting for buffers to come home. referenced_images: %d", + self->referenced_images); + } + + /* this will possibly invalidate any AImge currently in the pipeline */ g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); if (AImageReader_new (width, height, fmt, self->max_images, @@ -1172,11 +1190,14 @@ gst_ahc2_src_set_caps (GstBaseSrc * src, GstCaps * caps) ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, NULL, 1, &self->capture_request, NULL); + GST_OBJECT_UNLOCK (self); + return TRUE; failed: - g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); + GST_OBJECT_UNLOCK (self); + g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); g_clear_pointer (&self->camera_output_target, (GDestroyNotify) ACameraOutputTarget_free); @@ -1193,17 +1214,17 @@ gst_ahc2_src_get_caps (GstBaseSrc * src, GstCaps * filter) ACameraMetadata *metadata = NULL; ACameraMetadata_const_entry entry, fps_entry; + GST_OBJECT_LOCK (self); + if (ACameraManager_getCameraCharacteristics (self->camera_manager, self->camera_id_list->cameraIds[self->camera_index], &metadata) != ACAMERA_OK) { GST_ERROR_OBJECT (self, "Failed to get metadata from camera device"); + GST_OBJECT_UNLOCK (self); return caps; } - - GST_OBJECT_LOCK (self); - #ifndef GST_DISABLE_GST_DEBUG if (ACameraMetadata_getConstEntry (metadata, ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL, &entry) == ACAMERA_OK) { @@ -1333,8 +1354,8 @@ gst_ahc2_src_get_caps (GstBaseSrc * src, GstCaps * filter) GST_WARNING_OBJECT (self, " No available stream configurations detected"); } - g_clear_pointer (&format, gst_structure_free); - g_clear_pointer (&metadata, (GDestroyNotify) ACameraMetadata_free); + gst_structure_free (format); + ACameraMetadata_free (metadata); GST_DEBUG_OBJECT (self, "%" GST_PTR_FORMAT, caps); @@ -1348,8 +1369,11 @@ gst_ahc2_src_unlock (GstBaseSrc * src) { GstAHC2Src *self = GST_AHC2_SRC (src); - GST_DEBUG_OBJECT (self, "unlocking create"); + GST_DEBUG_OBJECT (self, "flushing the queue"); + GST_OBJECT_LOCK (self); gst_data_queue_set_flushing (self->outbound_queue, TRUE); + gst_data_queue_flush (self->outbound_queue); + GST_OBJECT_UNLOCK (self); return TRUE; } @@ -1359,8 +1383,11 @@ gst_ahc2_src_unlock_stop (GstBaseSrc * src) { GstAHC2Src *self = GST_AHC2_SRC (src); - GST_DEBUG_OBJECT (self, "stopping unlock"); + GST_DEBUG_OBJECT (self, "stop flushing the queue"); + GST_OBJECT_LOCK (self); gst_data_queue_set_flushing (self->outbound_queue, FALSE); + gst_data_queue_flush (self->outbound_queue); + GST_OBJECT_UNLOCK (self); return TRUE; } @@ -1372,11 +1399,12 @@ gst_ahc2_src_create (GstPushSrc * src, GstBuffer ** buffer) GstDataQueueItem *item; if (!gst_data_queue_pop (self->outbound_queue, &item)) { - GST_DEBUG_OBJECT (self, "We're flushing"); + GST_DEBUG_OBJECT (self, "We're flushing, item: %p", item); return GST_FLOW_FLUSHING; } *buffer = GST_BUFFER (item->object); + GST_TRACE_OBJECT (self, "Create function for buffer: %p", *buffer); g_free (item); return GST_FLOW_OK; @@ -1453,7 +1481,7 @@ static gboolean gst_ahc2_src_start (GstBaseSrc * src) { GstAHC2Src *self = GST_AHC2_SRC (src); - g_mutex_lock (&self->mutex); + GST_OBJECT_LOCK (self); if (!gst_ahc2_src_camera_open (self)) { goto out; @@ -1463,7 +1491,7 @@ gst_ahc2_src_start (GstBaseSrc * src) self->started = TRUE; out: - g_mutex_unlock (&self->mutex); + GST_OBJECT_UNLOCK (self); return self->started; } @@ -1473,14 +1501,14 @@ gst_ahc2_src_stop (GstBaseSrc * src) { GstAHC2Src *self = GST_AHC2_SRC (src); - g_mutex_lock (&self->mutex); + GST_OBJECT_LOCK (self); ACameraCaptureSession_abortCaptures (self->camera_capture_session); gst_ahc2_src_camera_close (self); gst_data_queue_flush (self->outbound_queue); self->started = FALSE; - g_mutex_unlock (&self->mutex); + GST_OBJECT_UNLOCK (self); return TRUE; } @@ -1648,14 +1676,13 @@ gst_ahc2_src_finalize (GObject * object) g_clear_pointer (&self->outbound_queue, gst_object_unref); - g_mutex_clear (&self->mutex); G_OBJECT_CLASS (parent_class)->finalize (object); } static void data_queue_item_free (GstDataQueueItem * item) { - g_clear_pointer (&item->object, (GDestroyNotify) gst_mini_object_unref); + gst_mini_object_unref (item->object); g_free (item); } @@ -1682,11 +1709,13 @@ image_reader_on_image_available (void *context, AImageReader * reader) gst_object_unref (clock); } + GST_OBJECT_LOCK (self); + if (AImageReader_acquireLatestImage (reader, &image) != AMEDIA_OK) { GST_DEBUG_OBJECT (self, "No image available"); + GST_OBJECT_UNLOCK (self); return; } - g_mutex_lock (&self->mutex); GST_DEBUG_OBJECT (self, "Acquired an image (ts: %" GST_TIME_FORMAT ")", GST_TIME_ARGS (current_ts)); @@ -1710,7 +1739,7 @@ image_reader_on_image_available (void *context, AImageReader * reader) AImage_delete (image); GST_DEBUG_OBJECT (self, "Droping image (reason: %s)", self->started ? "first frame" : "not yet started"); - g_mutex_unlock (&self->mutex); + GST_OBJECT_UNLOCK (self); return; } @@ -1742,25 +1771,30 @@ image_reader_on_image_available (void *context, AImageReader * reader) gst_buffer_append_memory (buffer, mem); } + g_atomic_int_add (&self->referenced_images, 1); + GST_TRACE_OBJECT (self, "Created image:%p, referenced_images: %d", image, + self->referenced_images); + item = g_new0 (GstDataQueueItem, 1); item->object = GST_MINI_OBJECT (buffer); item->size = gst_buffer_get_size (buffer); item->visible = TRUE; item->destroy = (GDestroyNotify) data_queue_item_free; + GST_TRACE_OBJECT (self, "About to push buffer: %p on queue", buffer); if (!gst_data_queue_push (self->outbound_queue, item)) { item->destroy (item); GST_DEBUG_OBJECT (self, "Failed to push item because we're flushing"); } gst_wrapped_aimage_unref (wrapped_aimage); - g_mutex_unlock (&self->mutex); + GST_OBJECT_UNLOCK (self); GST_TRACE_OBJECT (self, - "created buffer from image callback %" G_GSIZE_FORMAT ", ts %" + "created buffer %p from image callback %" G_GSIZE_FORMAT ", ts %" GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT ", offset %" G_GINT64_FORMAT ", offset_end %" G_GINT64_FORMAT, - gst_buffer_get_size (buffer), + buffer, gst_buffer_get_size (buffer), GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer)), GST_TIME_ARGS (GST_BUFFER_DURATION (buffer)), GST_BUFFER_OFFSET (buffer), GST_BUFFER_OFFSET_END (buffer)); @@ -2052,8 +2086,6 @@ gst_ahc2_src_init (GstAHC2Src * self) self->image_reader_listener.onImageAvailable = image_reader_on_image_available; - g_mutex_init (&self->mutex); - #ifndef GST_DISABLE_GST_DEBUG { int idx = 0; From dfec92074318f441eac23d13de5cd9d0cfe8801b Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 30 Aug 2022 15:04:17 +0200 Subject: [PATCH 247/339] amcvideoenc: fix mess around level and profile for H.264 Also specify the alignment as part of caps always. --- .../sys/androidmedia/gstamcvideoenc.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c index d04a1d2684c..6ec5e0cda59 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c @@ -181,6 +181,10 @@ create_amc_format (GstAmcVideoEnc * encoder, GstVideoCodecState * input_state, profile_string = gst_structure_get_string (s, "profile"); level_string = gst_structure_get_string (s, "level"); + GST_DEBUG_OBJECT (encoder, + "Creating GstAmcFormat for profile: %s and level: %s", + profile_string, level_string); + if (strcmp (name, "video/mpeg") == 0) { gint mpegversion; @@ -277,14 +281,20 @@ create_amc_format (GstAmcVideoEnc * encoder, GstVideoCodecState * input_state, /* FIXME: Set to any value in AVCProfile* leads to * codec configuration fail */ - /* gst_amc_format_set_int (format, amc_profile.key, 0x40); */ + GST_DEBUG_OBJECT (encoder, "Setting profile-id to %d", amc_profile.id); + gst_amc_format_set_int (format, amc_profile.key, amc_profile.id, &err); + if (err) + GST_ELEMENT_WARNING_FROM_ERROR (encoder, err); } if (level_string) { if (amc_level.id == -1) goto unsupported_level; - /* gst_amc_format_set_int (format, amc_level.key, amc_level.id); */ + GST_DEBUG_OBJECT (encoder, "Setting level-id to %d", amc_level.id); + gst_amc_format_set_int (format, amc_level.key, amc_level.id, &err); + if (err) + GST_ELEMENT_WARNING_FROM_ERROR (encoder, err); } /* On Android N_MR1 and higher, i-frame-interval can be a float value */ @@ -427,7 +437,7 @@ caps_from_amc_format (GstAmcFormat * amc_format) } if (gst_amc_format_get_int (amc_format, "level", &amc_level, NULL)) { - level_string = gst_amc_avc_level_to_string (amc_profile); + level_string = gst_amc_avc_level_to_string (amc_level); if (!level_string) goto unsupported_level; @@ -471,7 +481,8 @@ caps_from_amc_format (GstAmcFormat * amc_format) gst_caps_set_simple (caps, "width", G_TYPE_INT, width, "height", G_TYPE_INT, height, - "framerate", GST_TYPE_FRACTION, fraction_n, fraction_d, NULL); + "framerate", GST_TYPE_FRACTION, fraction_n, fraction_d, + "alignment", G_TYPE_STRING, "au", NULL); g_free (mime); return caps; From 22b5f49ac105cb3d3bdd05bc5364ea27a447f2a1 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 30 Aug 2022 17:48:18 +0200 Subject: [PATCH 248/339] ahc2src: improve the memory-handling of AImage to GstBuffer --- .../sys/androidmedia/gstahc2src.c | 146 ++++++++++++++---- 1 file changed, 113 insertions(+), 33 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c index 9eca4b2f291..02677416aeb 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c @@ -62,6 +62,9 @@ struct _GstAHC2Src gboolean started; gint max_images; + gint width; + gint height; + volatile gint referenced_images; gint camera_index; @@ -1067,7 +1070,7 @@ gst_ahc2_src_set_caps (GstBaseSrc * src, GstCaps * caps) gint fmt = -1; const gchar *format_str = NULL; const GValue *framerate_value; - gint width, height, fps_min = 0, fps_max = 0; + gint fps_min = 0, fps_max = 0; ANativeWindow *image_reader_window = NULL; @@ -1080,8 +1083,8 @@ gst_ahc2_src_set_caps (GstBaseSrc * src, GstCaps * caps) format_str = gst_structure_get_string (s, "format"); format = gst_video_format_from_string (format_str); - gst_structure_get_int (s, "width", &width); - gst_structure_get_int (s, "height", &height); + gst_structure_get_int (s, "width", &self->width); + gst_structure_get_int (s, "height", &self->height); framerate_value = gst_structure_get_value (s, "framerate"); if (GST_VALUE_HOLDS_FRACTION_RANGE (framerate_value)) { @@ -1136,11 +1139,12 @@ gst_ahc2_src_set_caps (GstBaseSrc * src, GstCaps * caps) /* this will possibly invalidate any AImge currently in the pipeline */ g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); - if (AImageReader_new (width, height, fmt, self->max_images, + if (AImageReader_new (self->width, self->height, fmt, self->max_images, &self->image_reader) != AMEDIA_OK) { GST_ERROR_OBJECT (self, "One or more of parameter " "(width: %d, height: %d, format: %s, max-images: %d) " - "is not supported.", width, height, format_str, self->max_images); + "is not supported.", self->width, self->height, format_str, + self->max_images); goto failed; } @@ -1686,6 +1690,109 @@ data_queue_item_free (GstDataQueueItem * item) g_free (item); } +#if 0 +ACameraMetadata *metadata = NULL; +if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + return; +} +ACameraMetadata_const_entry orientation; +ACameraMetadata_getConstEntry (metadata, ACAMERA_SENSOR_ORIENTATION, + &orientation); +int32_t angle = orientation.data.i32[0]; +GST_DEBUG ("Our angle is: %d", angle); +#endif + +static void +_aimage_to_gstbuffer (GstAHC2Src * self, AImage * image, GstBuffer * buffer) +{ + GstWrappedAImage *wrapped_aimage; + + guint8 *y_data; + guint8 *u_data; + guint8 *v_data; + + gint y_length; + gint u_length; + gint v_length; + + gint y_stride; + gint u_stride; + gint v_stride; + + gint u_pixelstride; + gint v_pixelstride; + + GstMemory *y_mem; + GstMemory *uv_mem; + + gint n_planes; + + AImage_getNumberOfPlanes (image, &n_planes); + if (n_planes != 3) { + GST_ERROR_OBJECT (self, "Not getting 3 planes!!"); + g_assert_not_reached (); + } + + AImage_getPlaneData (image, 0, &y_data, &y_length); + AImage_getPlaneData (image, 1, &u_data, &u_length); + AImage_getPlaneData (image, 2, &v_data, &v_length); + + AImage_getPlaneRowStride (image, 0, &y_stride); + AImage_getPlaneRowStride (image, 1, &u_stride); + AImage_getPlaneRowStride (image, 2, &v_stride); + + AImage_getPlanePixelStride (image, 1, &u_pixelstride); + AImage_getPlanePixelStride (image, 2, &v_pixelstride); + + if (y_stride != self->width) { + GST_ERROR_OBJECT (self, + "Assumption about AImage being stride=width broken!"); + g_assert_not_reached (); + } + + wrapped_aimage = g_new0 (GstWrappedAImage, 1); + wrapped_aimage->refcount = 1; + wrapped_aimage->ahc2src = g_object_ref (self); + wrapped_aimage->image = image; + + /* append the Y memory */ + y_mem = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, + y_data, y_length, 0, y_length, + gst_wrapped_aimage_ref (wrapped_aimage), + (GDestroyNotify) gst_wrapped_aimage_unref); + gst_buffer_append_memory (buffer, y_mem); + + + /* if all of this is true, we can safly assume a NV12 layout, where the + whole memory for the UV plane is contained inside both u_data and v_data, + with only 1 pixel moved */ + if (u_length == v_length && /* we have the same length */ + u_stride == v_stride && /* we have the same stride */ + ((u_data + 1 == v_data) || (v_data + 1 == u_data)) && /* v_data is basically pointing one pixel into u_data (or v.v), otherwise identical, expected for NV12/NV21 */ + u_pixelstride == 2 && v_pixelstride == 2 && /* the packing you expect from inteleaved U/V/U/V in NV12 */ + u_length == y_length / u_pixelstride - 1) { /* when reading all the U pixels, you expect the length to be /2-1 */ + + uv_mem = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, + u_data, y_length / 2, 0, y_length / 2, + gst_wrapped_aimage_ref (wrapped_aimage), + (GDestroyNotify) gst_wrapped_aimage_unref); + gst_buffer_append_memory (buffer, uv_mem); + } else { + /* we need this assumption to be true for now! */ + GST_ERROR_OBJECT (self, "Not getting the NV12 memory-layout we expected: " + "y_length=%d u_length=%d, v_length=%d, " + "u_stride=%d, v_stride=%d, u_data=%p, v_data=%p, " + "u_pixelstride=%d, v_pixelstride=%d", + y_length, u_length, v_length, u_stride, v_stride, u_data, v_data, + u_pixelstride, v_pixelstride); + g_assert_not_reached (); + } + + gst_wrapped_aimage_unref (wrapped_aimage); +} + static void image_reader_on_image_available (void *context, AImageReader * reader) { @@ -1696,11 +1803,7 @@ image_reader_on_image_available (void *context, AImageReader * reader) GstClockTime duration = GST_CLOCK_TIME_NONE; GstClockTime current_ts = GST_CLOCK_TIME_NONE; GstClock *clock; - - GstWrappedAImage *wrapped_aimage; AImage *image = NULL; - gint n_planes; - gint i; if ((clock = GST_ELEMENT_CLOCK (self))) { GstClockTime base_time = GST_ELEMENT_CAST (self)->base_time; @@ -1743,33 +1846,11 @@ image_reader_on_image_available (void *context, AImageReader * reader) return; } - AImage_getNumberOfPlanes (image, &n_planes); - buffer = gst_buffer_new (); GST_BUFFER_DURATION (buffer) = duration; GST_BUFFER_PTS (buffer) = current_ts; - wrapped_aimage = g_new0 (GstWrappedAImage, 1); - wrapped_aimage->refcount = 1; - wrapped_aimage->ahc2src = g_object_ref (self); - wrapped_aimage->image = image; - - for (i = 0; i < n_planes; i++) { - guint8 *data; - gint length; - GstMemory *mem; - - AImage_getPlaneData (image, i, &data, &length); - - mem = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, - data, length, 0, length, - gst_wrapped_aimage_ref (wrapped_aimage), - (GDestroyNotify) gst_wrapped_aimage_unref); - - GST_TRACE_OBJECT (self, "Created a wrapped memory (ptr: %p, length: %d)", - mem, length); - gst_buffer_append_memory (buffer, mem); - } + _aimage_to_gstbuffer (self, image, buffer); g_atomic_int_add (&self->referenced_images, 1); GST_TRACE_OBJECT (self, "Created image:%p, referenced_images: %d", image, @@ -1787,7 +1868,6 @@ image_reader_on_image_available (void *context, AImageReader * reader) GST_DEBUG_OBJECT (self, "Failed to push item because we're flushing"); } - gst_wrapped_aimage_unref (wrapped_aimage); GST_OBJECT_UNLOCK (self); GST_TRACE_OBJECT (self, From 0808260dc196552fb0bff0b89304b7f56c746ead Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 5 Sep 2022 14:06:39 +0200 Subject: [PATCH 249/339] vpxenc: FIXME: revert upstream changes These made one of our vpx test fail. GST_DEBUG=*vp*:LOG GST_CHECKS=vpx_enc_rate_limit_key_frames python3 /pexip/external/meson/meson.py test vpx -vv Look into why and how more closely! --- .../gst-plugins-good/ext/vpx/gstvpxenc.c | 76 ++++--------------- 1 file changed, 15 insertions(+), 61 deletions(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c index 0777aded457..74bbb989a37 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c @@ -859,8 +859,8 @@ gst_vpx_enc_init (GstVPXEnc * gst_vpx_enc) gst_vpx_enc->cfg.g_profile = DEFAULT_PROFILE; - gst_video_encoder_set_min_force_key_unit_interval ( - GST_VIDEO_ENCODER (gst_vpx_enc), 1500 * GST_MSECOND); + gst_video_encoder_set_min_force_key_unit_interval (GST_VIDEO_ENCODER + (gst_vpx_enc), 1500 * GST_MSECOND); g_mutex_init (&gst_vpx_enc->encoder_lock); } @@ -1155,8 +1155,8 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, gst_vpx_enc->tile_columns = g_value_get_int (value); if (gst_vpx_enc_class->enable_tiles (gst_vpx_enc) && gst_vpx_enc->inited) { status = - vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_TILE_COLUMNS, - gst_vpx_enc->tile_columns); + vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_TILE_COLUMNS, + gst_vpx_enc->tile_columns); if (status != VPX_CODEC_OK) { GST_WARNING_OBJECT (gst_vpx_enc, "Failed to set VP9E_SET_TILE_COLUMNS: %s", @@ -1168,8 +1168,8 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, gst_vpx_enc->tile_rows = g_value_get_int (value); if (gst_vpx_enc_class->enable_tiles (gst_vpx_enc) && gst_vpx_enc->inited) { status = - vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_TILE_ROWS, - gst_vpx_enc->tile_rows); + vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_TILE_ROWS, + gst_vpx_enc->tile_rows); if (status != VPX_CODEC_OK) { GST_WARNING_OBJECT (gst_vpx_enc, "Failed to set VP9E_SET_TILE_ROWS: %s", @@ -1688,10 +1688,6 @@ gst_vpx_enc_destroy_encoder (GstVPXEnc * encoder) encoder->cfg.rc_twopass_stats_in.buf = NULL; encoder->cfg.rc_twopass_stats_in.sz = 0; } - - encoder->last_pts = GST_CLOCK_TIME_NONE; - encoder->last_input_duration = GST_CLOCK_TIME_NONE; - g_mutex_unlock (&encoder->encoder_lock); } @@ -1820,8 +1816,6 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, vpx_codec_destroy (&encoder->encoder); encoder->inited = FALSE; encoder->multipass_cache_idx++; - encoder->last_pts = GST_CLOCK_TIME_NONE; - encoder->last_input_duration = GST_CLOCK_TIME_NONE; } else { g_mutex_lock (&encoder->encoder_lock); } @@ -1928,20 +1922,19 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, gst_vpx_error_name (status)); } } - #ifdef HAVE_VPX_1_3 if (vpx_enc_class->enable_tiles (encoder)) { status = - vpx_codec_control (&encoder->encoder, VP9E_SET_TILE_COLUMNS, - encoder->tile_columns); + vpx_codec_control (&encoder->encoder, VP9E_SET_TILE_COLUMNS, + encoder->tile_columns); if (status != VPX_CODEC_OK) { GST_WARNING_OBJECT (encoder, "Failed to set VP9E_SET_TILE_COLUMNS: %s", gst_vpx_error_name (status)); } status = - vpx_codec_control (&encoder->encoder, VP9E_SET_TILE_ROWS, - encoder->tile_rows); + vpx_codec_control (&encoder->encoder, VP9E_SET_TILE_ROWS, + encoder->tile_rows); if (status != VPX_CODEC_OK) { GST_WARNING_OBJECT (encoder, "Failed to set VP9E_SET_TILE_ROWS: %s", gst_vpx_error_name (status)); @@ -2137,24 +2130,11 @@ gst_vpx_enc_process (GstVPXEnc * encoder) /* discard older frames that were dropped by libvpx */ frame = NULL; do { - GstClockTime pts_rt; - if (frame) gst_video_encoder_finish_frame (video_encoder, frame); frame = gst_video_encoder_get_oldest_frame (video_encoder); - if (!frame) { - GST_WARNING_OBJECT (encoder, - "vpx pts %" G_GINT64_FORMAT - " does not match input frames, discarding", pkt->data.frame.pts); - goto out; - } - - pts_rt = - gst_segment_to_running_time (&video_encoder->input_segment, - GST_FORMAT_TIME, frame->pts); - pts = - gst_util_uint64_scale (pts_rt, + gst_util_uint64_scale (frame->pts, encoder->cfg.g_timebase.den, encoder->cfg.g_timebase.num * (GstClockTime) GST_SECOND); GST_TRACE_OBJECT (encoder, "vpx pts: %" G_GINT64_FORMAT @@ -2219,8 +2199,6 @@ gst_vpx_enc_process (GstVPXEnc * encoder) pkt = vpx_codec_get_cx_data (&encoder->encoder, &iter); } - -out: g_mutex_unlock (&encoder->encoder_lock); return ret; @@ -2235,20 +2213,14 @@ gst_vpx_enc_drain (GstVideoEncoder * video_encoder) vpx_codec_err_t status; gint64 deadline; vpx_codec_pts_t pts; - GstClockTime gst_pts = 0; encoder = GST_VPX_ENC (video_encoder); g_mutex_lock (&encoder->encoder_lock); deadline = encoder->deadline; - if (GST_CLOCK_TIME_IS_VALID (encoder->last_pts)) - gst_pts = encoder->last_pts; - if (GST_CLOCK_TIME_IS_VALID (encoder->last_input_duration)) - gst_pts += encoder->last_input_duration; - pts = - gst_util_uint64_scale (gst_pts, + gst_util_uint64_scale (encoder->last_pts, encoder->cfg.g_timebase.den, encoder->cfg.g_timebase.num * (GstClockTime) GST_SECOND); @@ -2346,7 +2318,6 @@ gst_vpx_enc_handle_frame (GstVideoEncoder * video_encoder, int flags = 0; vpx_image_t *image; GstVideoFrame vframe; - GstClockTime pts_rt; vpx_codec_pts_t pts; unsigned long duration; GstVPXEncClass *vpx_enc_class; @@ -2371,28 +2342,11 @@ gst_vpx_enc_handle_frame (GstVideoEncoder * video_encoder, } g_mutex_lock (&encoder->encoder_lock); - - /* the input pts needs to be strictly increasing, see vpx_codec_encode() doc, so convert it to - * running time as we don't want to reset the encoder for each segment. */ - pts_rt = - gst_segment_to_running_time (&video_encoder->input_segment, - GST_FORMAT_TIME, frame->pts); - - /* vpx_codec_encode() enforces us to pass strictly increasing pts */ - if (GST_CLOCK_TIME_IS_VALID (encoder->last_pts) - && pts_rt <= encoder->last_pts) { - GST_WARNING_OBJECT (encoder, - "decreasing pts %" GST_TIME_FORMAT " previous buffer was %" - GST_TIME_FORMAT " enforce increasing pts", GST_TIME_ARGS (pts_rt), - GST_TIME_ARGS (encoder->last_pts)); - pts_rt = encoder->last_pts + 1; - } - pts = - gst_util_uint64_scale (pts_rt, + gst_util_uint64_scale (frame->pts, encoder->cfg.g_timebase.den, encoder->cfg.g_timebase.num * (GstClockTime) GST_SECOND); - encoder->last_pts = pts_rt; + encoder->last_pts = frame->pts; if (frame->duration != GST_CLOCK_TIME_NONE) { duration = @@ -2400,7 +2354,7 @@ gst_vpx_enc_handle_frame (GstVideoEncoder * video_encoder, encoder->cfg.g_timebase.num * (GstClockTime) GST_SECOND); if (duration > 0) { - encoder->last_input_duration = frame->duration; + encoder->last_pts += frame->duration; } else { /* We force the path ignoring the duration if we end up with a zero * value for duration after scaling (e.g. duration value too small) */ From e00f4c1776fca990cfa25bf39895986666a83854 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 7 Sep 2022 16:53:51 +0200 Subject: [PATCH 250/339] rtpsession: FIXUP don't set running_time to -1 for no latency --- subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c index 6f2faf47253..1d65fcd9614 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c @@ -2515,9 +2515,9 @@ gst_rtp_session_chain_send_rtp_common (GstRtpSession * rtpsession, if (priv->send_latency != GST_CLOCK_TIME_NONE) { running_time += priv->send_latency; } else { - GST_WARNING_OBJECT (rtpsession, + GST_INFO_OBJECT (rtpsession, "Can't determine running time for this packet without knowing configured latency"); - running_time = -1; + //running_time = -1; } } } else { From 02b7a4dc23e5fdd8d0f151bdf0a96a337f6c180c Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 23 Sep 2022 13:15:09 +0200 Subject: [PATCH 251/339] rtpsession: don't report same stats twice The notify is already on a timer, and notifying again outside this breaks the contract of not receiving more stats. --- subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 883760ee4b7..6dbba5de5a6 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -3433,7 +3433,7 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer, GstClockTime current_time, GstClockTime running_time, guint64 ntpnstime) { GstRTCPPacket packet; - gboolean more, is_bye = FALSE, do_sync = FALSE, has_report = FALSE; + gboolean more, is_bye = FALSE, do_sync = FALSE; RTPPacketInfo pinfo = { 0, }; GstFlowReturn result = GST_FLOW_OK; GstRTCPBuffer rtcp = { NULL, }; @@ -3464,11 +3464,9 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer, switch (type) { case GST_RTCP_TYPE_SR: - has_report = TRUE; rtp_session_process_sr (sess, &packet, &pinfo, &do_sync); break; case GST_RTCP_TYPE_RR: - has_report = TRUE; rtp_session_process_rr (sess, &packet, &pinfo); break; case GST_RTCP_TYPE_SDES: @@ -3518,10 +3516,6 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer, sess->stats.avg_rtcp_packet_size, pinfo.bytes); RTP_SESSION_UNLOCK (sess); - if (has_report) { - g_object_notify_by_pspec (G_OBJECT (sess), properties[PROP_STATS]); - } - pinfo.data = NULL; clean_packet_info (&pinfo); From fda2a312d0a403a158898525f109d6ab733ce48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Fri, 9 Sep 2022 22:29:09 +0200 Subject: [PATCH 252/339] vtenc: PEXHACK don't do B-frames from encoder if we can help it --- subprojects/gst-plugins-bad/sys/applemedia/vtenc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c index cbcba3e5b4d..ebfd6fc10b4 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c @@ -74,8 +74,8 @@ #include #define VTENC_DEFAULT_BITRATE 0 -#define VTENC_DEFAULT_FRAME_REORDERING TRUE -#define VTENC_DEFAULT_REALTIME FALSE +#define VTENC_DEFAULT_FRAME_REORDERING FALSE +#define VTENC_DEFAULT_REALTIME TRUE #define VTENC_DEFAULT_QUALITY 0.5 #define VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL 0 #define VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL_DURATION 0 From 46db001d00cb9d575744003149f6e1dffccd7ebb Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 26 Sep 2022 13:19:13 +0200 Subject: [PATCH 253/339] gstclock: FIXUP removal of weakref --- subprojects/gstreamer/gst/gstclock.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/subprojects/gstreamer/gst/gstclock.c b/subprojects/gstreamer/gst/gstclock.c index 1ef3a615fbf..df7bba3b315 100644 --- a/subprojects/gstreamer/gst/gstclock.c +++ b/subprojects/gstreamer/gst/gstclock.c @@ -168,8 +168,6 @@ struct _GstClockPrivate gint post_count; gboolean synced; - - GWeakRef *clock_weakref; }; typedef struct _GstClockEntryImpl GstClockEntryImpl; @@ -772,17 +770,6 @@ gst_clock_init (GstClock * clock) priv->timeout = DEFAULT_TIMEOUT; priv->times = g_new0 (GstClockTime, 4 * priv->window_size); priv->times_temp = priv->times + 2 * priv->window_size; - /* - * An atomically ref-counted wrapper around a GWeakRef for this GstClock, - * created by the clock and shared with all its clock entries. - * - * This exists because g_weak_ref_ operations are quite expensive and operate - * with a global GRWLock. _get takes a reader lock, _init and _clear take - * a writer lock. We want to avoid having to instantiate a new GWeakRef for - * every clock entry. - */ - priv->clock_weakref = g_atomic_rc_box_new (GWeakRef); - g_weak_ref_init (priv->clock_weakref, clock); } static void @@ -820,8 +807,6 @@ gst_clock_finalize (GObject * object) clock->priv->times_temp = NULL; GST_CLOCK_SLAVE_UNLOCK (clock); - g_atomic_rc_box_release_full (clock->priv->clock_weakref, - (GDestroyNotify) g_weak_ref_clear); g_mutex_clear (&clock->priv->slave_lock); g_cond_clear (&clock->priv->sync_cond); From 9810d5e19bacb190f03cf4dc2c42a2825dd8eeff Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 29 Sep 2022 17:08:56 +0200 Subject: [PATCH 254/339] gstelement: do not hold the element lock when activating the pad in gst_element_add_pad() - Fixes a deadlock in which we will hold the lock inside gst_element_add_pad() and then later in gst_element_post_message_default(). This is triggered by calling gst_pad_start_task() from the pads activate_mode() function. (cherry picked from commit 22cc9761fd8c352d18d1062dd6512e3b962a217c) --- .../gstreamer/tests/check/gst/gstelement.c | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/subprojects/gstreamer/tests/check/gst/gstelement.c b/subprojects/gstreamer/tests/check/gst/gstelement.c index a0e07f5199a..de5108ae988 100644 --- a/subprojects/gstreamer/tests/check/gst/gstelement.c +++ b/subprojects/gstreamer/tests/check/gst/gstelement.c @@ -1017,6 +1017,50 @@ GST_START_TEST (test_release_pads_during_dispose) GST_END_TEST; +static void +add_srcpad_deadlock_loop (void) +{ +} + +static gboolean +add_srcpad_deadlock_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + if (active) + return gst_pad_start_task (pad, (GstTaskFunction) add_srcpad_deadlock_loop, + NULL, NULL); + + return gst_pad_stop_task (pad); +} + +GST_START_TEST (test_add_srcpad_deadlock) +{ + GstTestElement3 *te3 = g_object_new (gst_test_element3_get_type (), NULL); + GstPadTemplate *pad_template; + GstPad *srcpad; + + pad_template = + gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (te3), + "src_%u"); + g_return_if_fail (pad_template != NULL); + + srcpad = gst_pad_new_from_template (pad_template, "src"); + g_return_if_fail (srcpad != NULL); + gst_pad_set_activatemode_function (srcpad, add_srcpad_deadlock_activate_mode); + + gst_element_set_state (GST_ELEMENT (te3), GST_STATE_PLAYING); + + /* gst_element_add_pad should activate the pad, causing activate_mode and + gst_pad_start_task to be called */ + gst_element_add_pad (GST_ELEMENT (te3), srcpad); + + gst_element_set_state (GST_ELEMENT (te3), GST_STATE_NULL); + + gst_object_unref (te3); +} + +GST_END_TEST; + static Suite * gst_element_suite (void) @@ -1037,6 +1081,7 @@ gst_element_suite (void) tcase_add_test (tc_chain, test_forbidden_pad_template_names); tcase_add_test (tc_chain, test_foreach_pad); tcase_add_test (tc_chain, test_release_pads_during_dispose); + tcase_add_test (tc_chain, test_add_srcpad_deadlock); return s; } From 7372adf228f359aacabbe04d0ca98f2fe3c4f396 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 6 Oct 2022 13:23:51 +0200 Subject: [PATCH 255/339] androidmedia: populate devices list before starting the GstAcamDeviceProvider --- .../sys/androidmedia/gstacamdeviceprovider.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c index 7ace1cbd3a6..a0a564b2f89 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c @@ -170,6 +170,22 @@ gst_acam_device_provider_probe (GstDeviceProvider * object) return devices; } +static void +gst_acam_device_provider_populate_devices (GstDeviceProvider * provider) +{ + GList *devices, *it; + + devices = gst_acam_device_provider_probe (provider); + + for (it = devices; it != NULL; it = g_list_next (it)) { + GstDevice *device = GST_DEVICE_CAST (it->data); + if (device) + gst_device_provider_device_add (provider, device); + } + + g_list_free_full (devices, gst_object_unref); +} + static gboolean gst_acam_device_provider_start (GstDeviceProvider * object) { @@ -183,6 +199,8 @@ gst_acam_device_provider_start (GstDeviceProvider * object) return FALSE; } + gst_acam_device_provider_populate_devices (object); + if (ACameraManager_registerAvailabilityCallback (provider->manager, &provider->listeners) != ACAMERA_OK) { GST_ERROR_OBJECT (provider, "Couldn't register Availability Callbacks"); From 7455be59b5de73b994db2dafa828d6023ff21a33 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 6 Oct 2022 13:51:08 +0200 Subject: [PATCH 256/339] gstahc2src: added camera_status_t logging --- .../sys/androidmedia/gstahc2src.c | 62 ++++++++++++++----- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c index 02677416aeb..900ca1be395 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c @@ -190,6 +190,36 @@ enum static guint signals[LAST_SIGNAL] = { 0 }; +#ifndef IF_EQUAL_RETURN +#define IF_EQUAL_RETURN(param, val) \ + if (val == param) \ + return #val +#endif + +static const gchar * +camera_status_t_to_string (camera_status_t s) +{ + IF_EQUAL_RETURN (s, ACAMERA_OK); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_BASE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_DEVICE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_DISABLED); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_DISCONNECTED); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_IN_USE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_SERVICE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_INVALID_OPERATION); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_INVALID_PARAMETER); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_MAX_CAMERA_IN_USE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_METADATA_NOT_FOUND); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_NOT_ENOUGH_MEMORY); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_PERMISSION_DENIED); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_SESSION_CLOSED); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_STREAM_CONFIGURE_FAIL); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_UNKNOWN); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_UNSUPPORTED_OPERATION); + return NULL; +} + + static GstPhotographyCaps gst_ahc2_src_get_capabilities (GstPhotography * photo) { @@ -1438,43 +1468,45 @@ static gboolean gst_ahc2_src_camera_open (GstAHC2Src * self) { gboolean ret = FALSE; + camera_status_t status; const gchar *camera_id; g_return_val_if_fail (self->camera_id_list != NULL, FALSE); - g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, FALSE); camera_id = self->camera_id_list->cameraIds[self->camera_index]; - GST_DEBUG_OBJECT (self, "Trying to open camera device (id: %s)", camera_id); - if (ACameraManager_openCamera (self->camera_manager, camera_id, - &self->device_state_cb, &self->camera_device) != ACAMERA_OK) { - - GST_ERROR_OBJECT (self, "Failed to open camera device (id: %s)", camera_id); + status = ACameraManager_openCamera (self->camera_manager, camera_id, + &self->device_state_cb, &self->camera_device); + if (status != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to open camera device (id: %s, status: %s)", + camera_id, camera_status_t_to_string (status)); goto out; } - if (ACameraDevice_createCaptureRequest (self->camera_device, - self->camera_template_type, &self->capture_request) != ACAMERA_OK) { - GST_ERROR_OBJECT (self, "Failed to create camera request (camera id: %s)", - camera_id); + status = ACameraDevice_createCaptureRequest (self->camera_device, + self->camera_template_type, &self->capture_request); + if (status != ACAMERA_OK) { + GST_ERROR_OBJECT (self, + "Failed to create camera request (camera id: %s, status: %s)", + camera_id, camera_status_t_to_string (status)); goto out; } - if (ACaptureSessionOutputContainer_create (&self->capture_sout_container) != - ACAMERA_OK) { + status = + ACaptureSessionOutputContainer_create (&self->capture_sout_container); + if (status != ACAMERA_OK) { GST_ERROR_OBJECT (self, - "Failed to create capture session output container (camera id: %s)", - camera_id); + "Failed to create capture session output container (camera id: %s, status: %s)", + camera_id, camera_status_t_to_string (status)); goto out; } ret = TRUE; out: - if (!ret) gst_ahc2_src_camera_close (self); From 738bba97ea62ec07880d3289fa1087758eed71a2 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 10 Oct 2022 11:36:10 +0200 Subject: [PATCH 257/339] amdeviceprovider: fixing device-id property default value --- .../gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c index ad768b006e8..d298a5dc079 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c @@ -797,7 +797,7 @@ gst_am_device_class_init (GstAmDeviceClass * klass) g_object_class_install_property (object_class, PROP_DEVICE_ID, g_param_spec_int ("device-id", "Audio Device ID", - "The audio device id", 0, G_MAXINT, -1, + "The audio device id", -1, G_MAXINT, -1, G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); } From be426fef1089f633f5258d1ad4314150bdd34157 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 14 Oct 2022 18:38:42 +0200 Subject: [PATCH 258/339] gstleaks: add "objects-alive-list" to the checkpoint --- .../gstreamer/plugins/tracers/gstleaks.c | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/subprojects/gstreamer/plugins/tracers/gstleaks.c b/subprojects/gstreamer/plugins/tracers/gstleaks.c index b586283496c..a229da994b3 100644 --- a/subprojects/gstreamer/plugins/tracers/gstleaks.c +++ b/subprojects/gstreamer/plugins/tracers/gstleaks.c @@ -355,6 +355,20 @@ object_log_free (ObjectLog * obj) g_free (obj); } +static guint +object_log_hash (const ObjectLog * obj) +{ + return (guint) obj->object + (guint) obj->type_qname; +} + +static gboolean +object_log_compare (const ObjectLog * a, const ObjectLog * b) +{ + if ((a->object == b->object) && (a->type_qname == b->type_qname)) + return TRUE; + return FALSE; +} + static void handle_object_destroyed (GstLeaksTracer * self, gpointer object, ObjectKind kind) @@ -1048,9 +1062,9 @@ gst_leaks_tracer_activity_start_tracking (GstLeaksTracer * self) return; } - self->added = g_hash_table_new_full (NULL, NULL, + self->added = g_hash_table_new_full (object_log_hash, object_log_compare, (GDestroyNotify) object_log_free, NULL); - self->removed = g_hash_table_new_full (NULL, NULL, + self->removed = g_hash_table_new_full (object_log_hash, object_log_compare, (GDestroyNotify) object_log_free, NULL); GST_OBJECT_UNLOCK (self); } @@ -1091,26 +1105,52 @@ process_checkpoint (GstTracerRecord * record, const gchar * record_type, } } + +static GHashTable * +_copy_and_remove (GHashTable * a, GHashTable * b) +{ + GHashTable *copy = g_hash_table_new (NULL, NULL); + GHashTableIter iter; + gpointer o; + + g_hash_table_iter_init (&iter, a); + while (g_hash_table_iter_next (&iter, &o, NULL)) { + ObjectLog *obj = o; + if (!g_hash_table_contains (b, o)) + g_hash_table_add (copy, o); + } + + return copy; +} + static GstStructure * gst_leaks_tracer_activity_get_checkpoint (GstLeaksTracer * self) { GValue added = G_VALUE_INIT; GValue removed = G_VALUE_INIT; + GValue alive = G_VALUE_INIT; GstStructure *s = gst_structure_new_empty ("activity-checkpoint"); g_value_init (&added, GST_TYPE_LIST); g_value_init (&removed, GST_TYPE_LIST); + g_value_init (&alive, GST_TYPE_LIST); GST_OBJECT_LOCK (self); + + GHashTable *copy = _copy_and_remove (self->added, self->removed); + process_checkpoint (tr_added, "objects-created", self->added, &added); process_checkpoint (tr_removed, "objects-removed", self->removed, &removed); + process_checkpoint (NULL, "objects-created", copy, &alive); g_hash_table_remove_all (self->added); g_hash_table_remove_all (self->removed); + g_hash_table_destroy (copy); GST_OBJECT_UNLOCK (self); gst_structure_take_value (s, "objects-created-list", &added); gst_structure_take_value (s, "objects-removed-list", &removed); + gst_structure_take_value (s, "objects-alive-list", &alive); return s; } From 1ac91a808ca871864be6604dd32dc094256780a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Mon, 17 Oct 2022 16:59:09 +0200 Subject: [PATCH 259/339] vp9dec: don't offer 10bit if libvpx does not support it --- subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c b/subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c index 3819e602a3d..6828c57a852 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c @@ -80,6 +80,7 @@ G_DEFINE_TYPE (GstVP9Dec, gst_vp9_dec, GST_TYPE_VPX_DEC); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vp9dec, "vp9dec", GST_RANK_PRIMARY, gst_vp9_dec_get_type (), vpx_element_init (plugin)); +#ifdef VPX_CODEC_CAP_HIGHBITDEPTH static GstCaps * gst_vp9_dec_get_src_caps (void) { @@ -90,6 +91,13 @@ gst_vp9_dec_get_src_caps (void) return gst_caps_from_string ((vpx_codec_get_caps (&vpx_codec_vp9_dx_algo) & VPX_CODEC_CAP_HIGHBITDEPTH) ? CAPS_HIGHBIT : CAPS_8BIT); } +#else +static GstCaps * +gst_vp9_dec_get_src_caps (void) +{ + return gst_caps_from_string (GST_VIDEO_CAPS_MAKE ("{ " GST_VP9_DEC_VIDEO_FORMATS_8BIT " }")); +} +#endif static void gst_vp9_dec_class_init (GstVP9DecClass * klass) From 929cde478c4274975e2c167215cf31b396b2a68f Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Sun, 23 Oct 2022 23:06:44 +0200 Subject: [PATCH 260/339] PEXHACK: revert gstvideorate to old version This to unbreak simulcast tests --- .../gst/videorate/gstvideorate.c | 394 +++++------------- 1 file changed, 115 insertions(+), 279 deletions(-) diff --git a/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c b/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c index 44bc68166a3..ec3a30e2588 100644 --- a/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c +++ b/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c @@ -100,7 +100,6 @@ enum #define DEFAULT_MAX_RATE G_MAXINT #define DEFAULT_RATE 1.0 #define DEFAULT_MAX_DUPLICATION_TIME 0 -#define DEFAULT_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION GST_SECOND enum { @@ -116,8 +115,7 @@ enum PROP_AVERAGE_PERIOD, PROP_MAX_RATE, PROP_RATE, - PROP_MAX_DUPLICATION_TIME, - PROP_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION + PROP_MAX_DUPLICATION_TIME }; static GstStaticPadTemplate gst_video_rate_src_template = @@ -297,27 +295,6 @@ gst_video_rate_class_init (GstVideoRateClass * klass) 0, G_MAXUINT64, DEFAULT_MAX_DUPLICATION_TIME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - /** - * GstVideoRate:max-closing-segment-duplication-duration: - * - * Limits the maximum duration for which the last buffer is duplicated when - * finalizing a segment or on EOS. When receiving an EOS event or a new - * segment, videorate duplicates the last frame to close the configured - * segment (copying the last buffer until its #GstSegment.stop time (or - * #GstSegment.start time for reverse playback) is reached), this property - * ensures that it won't push buffers covering a duration longer than - * specified. - * - * Since: 1.22 - */ - g_object_class_install_property (object_class, - PROP_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION, - g_param_spec_uint64 ("max-closing-segment-duplication-duration", - "Maximum closing segment duplication duration", - "Maximum duration of duplicated buffers to close current segment", 0, - G_MAXUINT64, DEFAULT_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - gst_element_class_set_static_metadata (element_class, "Video rate adjuster", "Filter/Effect/Video", "Drops/duplicates/adjusts timestamps on video frames to make a perfect stream", @@ -618,15 +595,21 @@ gst_video_rate_setcaps (GstBaseTransform * trans, GstCaps * in_caps, * https://github.com/pexip/mcu/commit/f819d921773ab222ee0fb950e9808c48b3794156 */ { - gint in_fps = (videorate->from_rate_numerator * 1000) / videorate->from_rate_denominator; - gint out_fps = (videorate->to_rate_numerator * 1000) / videorate->to_rate_denominator; + gint in_fps = + (videorate->from_rate_numerator * 1000) / + videorate->from_rate_denominator; + gint out_fps = + (videorate->to_rate_numerator * 1000) / videorate->to_rate_denominator; if (in_fps >= out_fps) videorate->drop_only = TRUE; } done: - if (ret) { - gst_caps_replace (&videorate->in_caps, in_caps); - } + /* After a setcaps, our caps may have changed. In that case, we can't use + * the old buffer, if there was one (it might have different dimensions) */ + GST_DEBUG_OBJECT (videorate, "swapping old buffers"); + gst_video_rate_swap_prev (videorate, NULL, GST_CLOCK_TIME_NONE); + videorate->last_ts = GST_CLOCK_TIME_NONE; + videorate->average = 0; return ret; @@ -639,7 +622,7 @@ gst_video_rate_setcaps (GstBaseTransform * trans, GstCaps * in_caps, } static void -gst_video_rate_reset (GstVideoRate * videorate, gboolean on_flush) +gst_video_rate_reset (GstVideoRate * videorate) { GST_DEBUG_OBJECT (videorate, "resetting internal variables"); @@ -654,10 +637,6 @@ gst_video_rate_reset (GstVideoRate * videorate, gboolean on_flush) videorate->discont = TRUE; videorate->average = 0; videorate->force_variable_rate = FALSE; - if (!on_flush) { - /* Do not clear caps on flush events as those are still valid */ - gst_clear_caps (&videorate->in_caps); - } gst_video_rate_swap_prev (videorate, NULL, 0); gst_segment_init (&videorate->segment, GST_FORMAT_TIME); @@ -666,7 +645,7 @@ gst_video_rate_reset (GstVideoRate * videorate, gboolean on_flush) static void gst_video_rate_init (GstVideoRate * videorate) { - gst_video_rate_reset (videorate, FALSE); + gst_video_rate_reset (videorate); videorate->silent = DEFAULT_SILENT; videorate->new_pref = DEFAULT_NEW_PREF; videorate->drop_only = DEFAULT_DROP_ONLY; @@ -676,8 +655,6 @@ gst_video_rate_init (GstVideoRate * videorate) videorate->rate = DEFAULT_RATE; videorate->pending_rate = DEFAULT_RATE; videorate->max_duplication_time = DEFAULT_MAX_DUPLICATION_TIME; - videorate->max_closing_segment_duplication_duration = - DEFAULT_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION; videorate->from_rate_numerator = 0; videorate->from_rate_denominator = 0; @@ -690,7 +667,7 @@ gst_video_rate_init (GstVideoRate * videorate) /* @outbuf: (transfer full) needs to be writable */ static GstFlowReturn gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf, - gboolean duplicate, GstClockTime next_intime, gboolean invalid_duration) + gboolean duplicate, GstClockTime next_intime) { GstFlowReturn res; GstClockTime push_ts; @@ -748,7 +725,7 @@ gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf, videorate->to_rate_denominator * GST_SECOND, videorate->to_rate_numerator); GST_BUFFER_DURATION (outbuf) = videorate->next_ts - push_ts; - } else if (!invalid_duration) { + } else { /* There must always be a valid duration on prevbuf if rate > 0, * it is ensured in the transform_ip function */ g_assert (GST_BUFFER_PTS_IS_VALID (outbuf)); @@ -778,7 +755,7 @@ gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf, /* flush the oldest buffer */ static GstFlowReturn gst_video_rate_flush_prev (GstVideoRate * videorate, gboolean duplicate, - GstClockTime next_intime, gboolean invalid_duration) + GstClockTime next_intime) { GstBuffer *outbuf; @@ -789,8 +766,7 @@ gst_video_rate_flush_prev (GstVideoRate * videorate, gboolean duplicate, /* make sure we can write to the metadata */ outbuf = gst_buffer_make_writable (outbuf); - return gst_video_rate_push_buffer (videorate, outbuf, duplicate, next_intime, - invalid_duration); + return gst_video_rate_push_buffer (videorate, outbuf, duplicate, next_intime); /* WARNINGS */ eos_before_buffers: @@ -805,14 +781,9 @@ gst_video_rate_swap_prev (GstVideoRate * videorate, GstBuffer * buffer, gint64 time) { GST_LOG_OBJECT (videorate, "swap_prev: storing buffer %p in prev", buffer); - - gst_buffer_replace (&videorate->prevbuf, buffer); - /* Ensure that ->prev_caps always match ->prevbuf */ - if (!buffer) - gst_caps_replace (&videorate->prev_caps, NULL); - else if (videorate->prev_caps != videorate->in_caps) - gst_caps_replace (&videorate->prev_caps, videorate->in_caps); - + if (videorate->prevbuf) + gst_buffer_unref (videorate->prevbuf); + videorate->prevbuf = buffer != NULL ? gst_buffer_ref (buffer) : NULL; videorate->prev_ts = time; } @@ -828,136 +799,7 @@ gst_video_rate_notify_duplicate (GstVideoRate * videorate) g_object_notify_by_pspec ((GObject *) videorate, pspec_duplicate); } -static gboolean -gst_video_rate_check_duplicate_to_close_segment (GstVideoRate * videorate, - GstClockTime last_input_ts, gboolean is_first) -{ - GstClockTime next_stream_time = videorate->next_ts - videorate->segment.base; - GstClockTime max_closing_segment_duplication_duration = - videorate->max_closing_segment_duplication_duration; - - if (!GST_CLOCK_TIME_IS_VALID (videorate->next_ts)) - return FALSE; - - if (videorate->segment.rate > 0.0) { - - if (!GST_CLOCK_TIME_IS_VALID (videorate->segment.stop)) { - /* Ensure that if no 'stop' is set, we push the last frame anyway */ - return is_first; - } - - if (next_stream_time >= videorate->segment.stop) - return FALSE; - - if (GST_CLOCK_TIME_IS_VALID (max_closing_segment_duplication_duration)) { - if (last_input_ts > videorate->next_ts) - return TRUE; - - return (videorate->next_ts - last_input_ts < - max_closing_segment_duplication_duration); - } - - return TRUE; - } - - /* Reverse playback */ - - if (!GST_CLOCK_TIME_IS_VALID (videorate->segment.start)) { - /* Ensure that if no 'start' is set, we push the last frame anyway */ - return is_first; - } - - if (next_stream_time < videorate->segment.start) - return FALSE; - - if (GST_CLOCK_TIME_IS_VALID (max_closing_segment_duplication_duration)) { - if (last_input_ts < videorate->next_ts) - return TRUE; - - return (last_input_ts - videorate->next_ts < - max_closing_segment_duplication_duration); - } - - return TRUE; -} - -static gint -gst_video_rate_duplicate_to_close_segment (GstVideoRate * videorate) -{ - gint count = 0; - GstFlowReturn res; - GstClockTime last_input_ts = videorate->prev_ts; - - if (videorate->drop_only) - return count; - - if (!videorate->prevbuf) { - GST_INFO_OBJECT (videorate, "got EOS before any buffer was received"); - - return count; - } - - GST_DEBUG_OBJECT (videorate, "Pushing buffers to close segment"); - - res = GST_FLOW_OK; - /* fill up to the end of current segment */ - while (res == GST_FLOW_OK - && gst_video_rate_check_duplicate_to_close_segment (videorate, - last_input_ts, count < 1)) { - res = - gst_video_rate_flush_prev (videorate, count > 0, GST_CLOCK_TIME_NONE, - FALSE); - - count++; - } - GST_DEBUG_OBJECT (videorate, "----> Pushed %d buffers to close segment", - count); - - return count; -} - -/* WORKAROUND: This works around BaseTransform limitation as instead of rolling - * back caps, we should be able to push caps only when we are sure we are ready - * to do so. Right now, BaseTransform doesn't let us do anything like that - * so we rollback to previous caps when strictly required (though we now it - * might not be so safe). - * - * To be used only when wanting to 'close' a segment, this function will reset - * caps to previous caps, which will match the content of `prevbuf` in that case - * - * Returns: The previous GstCaps if we rolled back to previous buffers, NULL - * otherwise. - * - * NOTE: When some caps are returned, we should reset them back after - * closing the segment is done. - */ -static GstCaps * -gst_video_rate_rollback_to_prev_caps_if_needed (GstVideoRate * videorate) -{ - GstCaps *prev_caps = NULL; - - if (videorate->prev_caps && videorate->prev_caps != videorate->in_caps) { - if (videorate->in_caps) - prev_caps = gst_caps_ref (videorate->in_caps); - - if (!gst_pad_send_event (GST_BASE_TRANSFORM_SINK_PAD (videorate), - gst_event_new_caps (videorate->prev_caps) - )) { - - GST_WARNING_OBJECT (videorate, "Could not send previous caps to close " - " segment, not closing it"); - - gst_video_rate_swap_prev (videorate, NULL, GST_CLOCK_TIME_NONE); - videorate->last_ts = GST_CLOCK_TIME_NONE; - videorate->average = 0; - } - - gst_clear_caps (&videorate->prev_caps); - } - - return prev_caps; -} - +#define MAGIC_LIMIT 25 static gboolean gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) { @@ -970,27 +812,62 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) { GstSegment segment; gint seqnum; - GstCaps *rolled_back_caps; + GstClockTime base_ts, next_ts; + gboolean reset_base_ts = FALSE; gst_event_copy_segment (event, &segment); if (segment.format != GST_FORMAT_TIME) goto format_error; + GST_DEBUG_OBJECT (videorate, "handle SEGMENT event"); + + /* We just want to update the accumulated stream_time */ + segment.start = (gint64) (segment.start / videorate->rate); segment.position = (gint64) (segment.position / videorate->rate); if (GST_CLOCK_TIME_IS_VALID (segment.stop)) segment.stop = (gint64) (segment.stop / videorate->rate); segment.time = (gint64) (segment.time / videorate->rate); + base_ts = gst_segment_position_from_running_time (&segment, + GST_FORMAT_TIME, + gst_segment_to_running_time (&videorate->segment, GST_FORMAT_TIME, + videorate->base_ts)); + next_ts = gst_segment_position_from_running_time (&segment, + GST_FORMAT_TIME, + gst_segment_to_running_time (&videorate->segment, GST_FORMAT_TIME, + videorate->next_ts)); - if (!gst_segment_is_equal (&segment, &videorate->segment)) { - rolled_back_caps = - gst_video_rate_rollback_to_prev_caps_if_needed (videorate); + /* Reset if the segment is discontinuous */ + if (next_ts == GST_CLOCK_TIME_NONE) { + reset_base_ts = TRUE; /* close up the previous segment, if appropriate */ if (videorate->prevbuf) { - /* fill up to the end of current segment */ - gint count = gst_video_rate_duplicate_to_close_segment (videorate); + gint count = 0; + GstFlowReturn res; + + res = GST_FLOW_OK; + /* fill up to the end of current segment, + * or only send out the stored buffer if there is no specific stop. + * regardless, prevent going loopy in strange cases */ + while (res == GST_FLOW_OK && count <= MAGIC_LIMIT + && !videorate->drop_only + && ((videorate->segment.rate > 0.0 + && GST_CLOCK_TIME_IS_VALID (videorate->segment.stop) + && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) + && videorate->next_ts - videorate->segment.base < + videorate->segment.stop) || (videorate->segment.rate < 0.0 + && GST_CLOCK_TIME_IS_VALID (videorate->segment.start) + && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) + && videorate->next_ts - videorate->segment.base >= + videorate->segment.start) + || count < 1)) { + res = + gst_video_rate_flush_prev (videorate, count > 0, + GST_CLOCK_TIME_NONE); + count++; + } if (count > 1) { videorate->dup += count - 1; if (!videorate->silent) @@ -999,39 +876,21 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) /* clean up for the new one; _chain will resume from the new start */ gst_video_rate_swap_prev (videorate, NULL, 0); } - - if (rolled_back_caps) { - GST_DEBUG_OBJECT (videorate, - "Resetting rolled back caps %" GST_PTR_FORMAT, rolled_back_caps); - if (!gst_pad_send_event (GST_BASE_TRANSFORM_SINK_PAD (videorate), - gst_event_new_caps (rolled_back_caps) - )) { - - GST_WARNING_OBJECT (videorate, - "Could not resend caps after closing " " segment"); - - GST_ELEMENT_ERROR (videorate, CORE, NEGOTIATION, - ("Could not resend caps after closing segment"), (NULL)); - gst_caps_unref (rolled_back_caps); - - return FALSE; - } - - gst_caps_unref (rolled_back_caps); - } + } else if (base_ts == GST_CLOCK_TIME_NONE) { + reset_base_ts = TRUE; } - videorate->base_ts = 0; - videorate->out_frame_count = 0; - videorate->next_ts = GST_CLOCK_TIME_NONE; - - /* We just want to update the accumulated stream_time */ + if (reset_base_ts) { + base_ts = 0; + videorate->out_frame_count = 0; + } + videorate->next_ts = next_ts; + videorate->base_ts = base_ts; gst_segment_copy_into (&segment, &videorate->segment); GST_DEBUG_OBJECT (videorate, "updated segment: %" GST_SEGMENT_FORMAT, &videorate->segment); - seqnum = gst_event_get_seqnum (event); gst_event_unref (event); event = gst_event_new_segment (&segment); @@ -1043,68 +902,56 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) case GST_EVENT_EOS:{ gint count = 0; GstFlowReturn res = GST_FLOW_OK; - GstCaps *rolled_back_caps; GST_DEBUG_OBJECT (videorate, "Got %s", gst_event_type_get_name (GST_EVENT_TYPE (event))); - rolled_back_caps = - gst_video_rate_rollback_to_prev_caps_if_needed (videorate); - /* If the segment has a stop position, fill the segment */ if (GST_CLOCK_TIME_IS_VALID (videorate->segment.stop)) { - /* fill up to the end of current segment */ - count = gst_video_rate_duplicate_to_close_segment (videorate); + /* fill up to the end of current segment, + * or only send out the stored buffer if there is no specific stop. + * regardless, prevent going loopy in strange cases */ + while (res == GST_FLOW_OK && count <= MAGIC_LIMIT + && !videorate->drop_only + && ((videorate->segment.rate > 0.0 + && GST_CLOCK_TIME_IS_VALID (videorate->segment.stop) + && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) + && videorate->next_ts - videorate->segment.base < + videorate->segment.stop) || (videorate->segment.rate < 0.0 + && GST_CLOCK_TIME_IS_VALID (videorate->segment.start) + && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) + && videorate->next_ts - videorate->segment.base >= + videorate->segment.start) + )) { + res = gst_video_rate_flush_prev (videorate, count > 0, + GST_CLOCK_TIME_NONE); + count++; + } } else if (!videorate->drop_only && videorate->prevbuf) { /* Output at least one frame but if the buffer duration is valid, output * enough frames to use the complete buffer duration */ if (GST_BUFFER_DURATION_IS_VALID (videorate->prevbuf)) { - GstClockTime end_ts, duration = - GST_BUFFER_DURATION (videorate->prevbuf); + GstClockTime end_ts = + videorate->next_ts + GST_BUFFER_DURATION (videorate->prevbuf); - if (GST_CLOCK_TIME_IS_VALID - (videorate->max_closing_segment_duplication_duration)) - duration = - MIN (videorate->max_closing_segment_duplication_duration, - duration); - - end_ts = videorate->next_ts + duration; - while (res == GST_FLOW_OK && ((videorate->segment.rate > 0.0 + while (res == GST_FLOW_OK && count <= MAGIC_LIMIT && + ((videorate->segment.rate > 0.0 && GST_CLOCK_TIME_IS_VALID (videorate->segment.stop) && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) && videorate->next_ts - videorate->segment.base < end_ts) || count < 1)) { res = gst_video_rate_flush_prev (videorate, count > 0, - GST_CLOCK_TIME_NONE, FALSE); + GST_CLOCK_TIME_NONE); count++; } } else { - /* allow the duration to be invalid as there is no way to infer it if we - * received a single buffer and not output framerate was set. */ res = - gst_video_rate_flush_prev (videorate, FALSE, GST_CLOCK_TIME_NONE, - TRUE); + gst_video_rate_flush_prev (videorate, FALSE, GST_CLOCK_TIME_NONE); count = 1; } } - if (rolled_back_caps) { - GST_DEBUG_OBJECT (videorate, - "Resetting rolled back caps %" GST_PTR_FORMAT, rolled_back_caps); - - if (!gst_pad_send_event (GST_BASE_TRANSFORM_SINK_PAD (videorate), - gst_event_new_caps (rolled_back_caps) - )) { - - /* Not erroring out on EOS as it won't be too bad in any case */ - GST_WARNING_OBJECT (videorate, "Could not resend caps after closing " - " segment on EOS (ignoring the error)"); - } - - gst_caps_unref (rolled_back_caps); - } - if (count > 1) { videorate->dup += count - 1; if (!videorate->silent) @@ -1121,7 +968,7 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) case GST_EVENT_FLUSH_STOP: /* also resets the segment */ GST_DEBUG_OBJECT (videorate, "Got FLUSH_STOP"); - gst_video_rate_reset (videorate, TRUE); + gst_video_rate_reset (videorate); break; case GST_EVENT_GAP: /* no gaps after videorate, ignore the event */ @@ -1588,14 +1435,12 @@ gst_video_rate_do_max_duplicate (GstVideoRate * videorate, GstBuffer * buffer, * previous buffer */ if (videorate->segment.rate < 0.0) { while (videorate->next_ts > prevtime) { - gst_video_rate_flush_prev (videorate, *count > 0, GST_CLOCK_TIME_NONE, - FALSE); + gst_video_rate_flush_prev (videorate, *count > 0, GST_CLOCK_TIME_NONE); *count += 1; } } else { while (videorate->next_ts <= prevtime) { - gst_video_rate_flush_prev (videorate, *count > 0, GST_CLOCK_TIME_NONE, - FALSE); + gst_video_rate_flush_prev (videorate, *count > 0, GST_CLOCK_TIME_NONE); *count += 1; } } @@ -1657,18 +1502,6 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) videorate = GST_VIDEO_RATE (trans); - if (videorate->prev_caps != videorate->in_caps) { - /* After caps where set we didn't reset the state so we could close - * the segment from previous caps if necessary, we got a buffer after the - * new caps so we can reset now */ - GST_DEBUG_OBJECT (videorate, "Clearing old buffers now that we had a buffer" - " after receiving caps"); - gst_video_rate_swap_prev (videorate, NULL, GST_CLOCK_TIME_NONE); - gst_clear_caps (&videorate->prev_caps); - videorate->last_ts = GST_CLOCK_TIME_NONE; - videorate->average = 0; - } - /* make sure the denominators are not 0 */ if (videorate->from_rate_denominator == 0 || videorate->to_rate_denominator == 0) @@ -1706,6 +1539,10 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) goto invalid_buffer; } + if (!gst_segment_clip (&videorate->segment, GST_FORMAT_TIME, in_ts, + GST_CLOCK_TIME_NONE, NULL, NULL)) + goto outside_segment; + /* get the time of the next expected buffer timestamp, we use this when the * next buffer has -1 as a timestamp */ last_ts = videorate->last_ts; @@ -1787,7 +1624,7 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) * GstBaseTransform can get its reference back. */ if ((r = gst_video_rate_push_buffer (videorate, gst_buffer_ref (buffer), FALSE, - GST_CLOCK_TIME_NONE, FALSE)) != GST_FLOW_OK) { + GST_CLOCK_TIME_NONE)) != GST_FLOW_OK) { res = r; goto done; } @@ -1917,7 +1754,7 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) /* on error the _flush function posted a warning already */ if ((r = gst_video_rate_flush_prev (videorate, - count > 1, intime, FALSE)) != GST_FLOW_OK) { + count > 1, intime)) != GST_FLOW_OK) { res = r; goto done; } @@ -1974,19 +1811,26 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) res = GST_BASE_TRANSFORM_FLOW_DROPPED; goto done; } + +outside_segment: + { + GST_WARNING_OBJECT (videorate, "Got buffer outide segment, discarding it"); + res = GST_BASE_TRANSFORM_FLOW_DROPPED; + goto done; + } } static gboolean gst_video_rate_start (GstBaseTransform * trans) { - gst_video_rate_reset (GST_VIDEO_RATE (trans), FALSE); + gst_video_rate_reset (GST_VIDEO_RATE (trans)); return TRUE; } static gboolean gst_video_rate_stop (GstBaseTransform * trans) { - gst_video_rate_reset (GST_VIDEO_RATE (trans), FALSE); + gst_video_rate_reset (GST_VIDEO_RATE (trans)); return TRUE; } @@ -2040,10 +1884,6 @@ gst_video_rate_set_property (GObject * object, case PROP_MAX_DUPLICATION_TIME: videorate->max_duplication_time = g_value_get_uint64 (value); break; - case PROP_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION: - videorate->max_closing_segment_duplication_duration = - g_value_get_uint64 (value); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -2106,10 +1946,6 @@ gst_video_rate_get_property (GObject * object, case PROP_MAX_DUPLICATION_TIME: g_value_set_uint64 (value, videorate->max_duplication_time); break; - case PROP_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION: - g_value_set_uint64 (value, - videorate->max_closing_segment_duplication_duration); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -2130,4 +1966,4 @@ GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, videorate, "Adjusts video frames", - plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) + plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) \ No newline at end of file From cf5e50606fa7fbe5139c82b411b42920cf5f3d75 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Thu, 27 Oct 2022 02:05:54 +0000 Subject: [PATCH 261/339] gstleaks: FIXUP remove unused variable --- subprojects/gstreamer/plugins/tracers/gstleaks.c | 1 - 1 file changed, 1 deletion(-) diff --git a/subprojects/gstreamer/plugins/tracers/gstleaks.c b/subprojects/gstreamer/plugins/tracers/gstleaks.c index a229da994b3..97a117f8512 100644 --- a/subprojects/gstreamer/plugins/tracers/gstleaks.c +++ b/subprojects/gstreamer/plugins/tracers/gstleaks.c @@ -1115,7 +1115,6 @@ _copy_and_remove (GHashTable * a, GHashTable * b) g_hash_table_iter_init (&iter, a); while (g_hash_table_iter_next (&iter, &o, NULL)) { - ObjectLog *obj = o; if (!g_hash_table_contains (b, o)) g_hash_table_add (copy, o); } From b543bd0c63c965cd190cd46e893f5381c487fcd4 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 26 Oct 2022 13:35:38 +0200 Subject: [PATCH 262/339] gstamc: downgrading unsupported and unknowns messages to DEBUG --- .../gst-plugins-bad/sys/androidmedia/gstamc.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c index b4db71b3e28..196ea4ded7c 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c @@ -310,7 +310,7 @@ scan_codecs (GstPlugin * plugin) if (!accepted_color_formats (gst_codec_type, is_encoder)) { if (!ignore_unknown_color_formats) { gst_codec_info->gl_output_only = TRUE; - GST_WARNING + GST_DEBUG ("%s %s has unknown color formats, only direct rendering will be supported", gst_codec_type->mime, is_encoder ? "encoder" : "decoder"); } @@ -553,7 +553,7 @@ accepted_color_formats (GstAmcCodecType * type, gboolean is_encoder) } if (!found) { - GST_ERROR ("Unknown color format 0x%x, ignoring", type->color_formats[i]); + GST_DEBUG ("Unknown color format 0x%x, ignoring", type->color_formats[i]); } } @@ -2044,7 +2044,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, gst_amc_aac_profile_to_string (type->profile_levels[j].profile); if (!profile) { - GST_ERROR ("Unable to map AAC profile 0x%08x", + GST_DEBUG ("Unable to map AAC profile 0x%08x", type->profile_levels[j].profile); continue; } @@ -2096,7 +2096,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, "parsed", G_TYPE_BOOLEAN, TRUE, NULL); encoded_ret = gst_caps_merge_structure (encoded_ret, tmp); } else { - GST_WARNING ("Unsupported mimetype '%s'", type->mime); + GST_DEBUG ("Unsupported mimetype '%s'", type->mime); } } } else if (g_str_has_prefix (type->mime, "video/")) { @@ -2115,7 +2115,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, gst_amc_color_format_to_video_format (codec_info, type->mime, type->color_formats[j]); if (format == GST_VIDEO_FORMAT_UNKNOWN) { - GST_WARNING ("Unknown color format 0x%08x for codec %s", + GST_DEBUG ("Unknown color format 0x%08x for codec %s", type->color_formats[j], type->mime); continue; } @@ -2292,7 +2292,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, profile, &alternative); if (!profile) { - GST_ERROR ("Unable to map H264 profile 0x%08x", + GST_DEBUG ("Unable to map H264 profile 0x%08x", type->profile_levels[j].profile); continue; } @@ -2368,7 +2368,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, profile); if (!profile) { - GST_ERROR ("Unable to map H265 profile 0x%08x", + GST_DEBUG ("Unable to map H265 profile 0x%08x", type->profile_levels[j].profile); continue; } @@ -2449,7 +2449,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, encoded_ret = gst_caps_merge_structure (encoded_ret, tmp); } else { - GST_WARNING ("Unsupported mimetype '%s'", type->mime); + GST_DEBUG ("Unsupported mimetype '%s'", type->mime); } } } From 789deefa6dc786649c2cf96372ae55e87ac700e6 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 1 Nov 2022 15:21:03 +0100 Subject: [PATCH 263/339] gstamc-codec-jni: added NULL check before unrefing codec->surface --- .../gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c b/subprojects/gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c index dcc1314f64e..9e0c6288ab6 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c @@ -586,7 +586,9 @@ gst_amc_codec_configure (GstAmcCodec * codec, GstAmcFormat * format, env = gst_amc_jni_get_env (); if (surface) { - g_object_unref (codec->surface); + if (codec->surface) + g_object_unref (codec->surface); + codec->surface = gst_amc_surface_new ((GstAmcSurfaceTextureJNI *) surface, err); if (!codec->surface) From e607042ebf0b9f1c7e02b8aa1ed901516de9b274 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 1 Nov 2022 15:35:26 +0100 Subject: [PATCH 264/339] gstamcvideoenc: avoid flushing the encoder twice if we have already flushed - Fixes a java exception of flushing the encoder in an ilegal state --- .../gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c index 6ec5e0cda59..7bb27a01fe4 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c @@ -776,10 +776,12 @@ gst_amc_video_enc_change_state (GstElement * element, GstStateChange transition) case GST_STATE_CHANGE_PAUSED_TO_PLAYING: break; case GST_STATE_CHANGE_PAUSED_TO_READY: - self->flushing = TRUE; - gst_amc_codec_flush (self->codec, &err); - if (err) - GST_ELEMENT_WARNING_FROM_ERROR (self, err); + if (!self->flushing) { + self->flushing = TRUE; + gst_amc_codec_flush (self->codec, &err); + if (err) + GST_ELEMENT_WARNING_FROM_ERROR (self, err); + } g_mutex_lock (&self->drain_lock); self->draining = FALSE; g_cond_broadcast (&self->drain_cond); From d2734c096befc105638ae08b606854e5cbc30a8a Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 3 Nov 2022 11:29:08 +0100 Subject: [PATCH 265/339] ahc2src: logging camera error messages --- .../gst-plugins-bad/sys/androidmedia/gstahc2src.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c index 900ca1be395..7c747cc7f8a 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c @@ -219,6 +219,16 @@ camera_status_t_to_string (camera_status_t s) return NULL; } +static const gchar * +camera_error_to_string (int e) +{ + IF_EQUAL_RETURN (e, ERROR_CAMERA_IN_USE); + IF_EQUAL_RETURN (e, ERROR_MAX_CAMERAS_IN_USE); + IF_EQUAL_RETURN (e, ERROR_CAMERA_DISABLED); + IF_EQUAL_RETURN (e, ERROR_CAMERA_DEVICE); + IF_EQUAL_RETURN (e, ERROR_CAMERA_SERVICE); + return NULL; +} static GstPhotographyCaps gst_ahc2_src_get_capabilities (GstPhotography * photo) @@ -1926,8 +1936,8 @@ camera_device_on_error (void *context, ACameraDevice * device, int error) { GstAHC2Src *self = GST_AHC2_SRC (context); - GST_ERROR_OBJECT (self, "Error (code: %d) on Camera[%s]", error, - ACameraDevice_getId (device)); + GST_ERROR_OBJECT (self, "Error (err: %s, code: %d) on Camera[%s]", + camera_error_to_string (error), error, ACameraDevice_getId (device)); } static void From b607c4ce595e2eb5d3b19fb14f443aade1e7fae7 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 3 Nov 2022 12:18:08 +0100 Subject: [PATCH 266/339] ahc2src: add PROP_EXPOSURE_MODE stub to avoid error of not implementing GstPhotography interface --- .../sys/androidmedia/gstahc2src.c | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c index 7c747cc7f8a..b62af9c7983 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c @@ -174,10 +174,11 @@ typedef enum PROP_WHITE_BALANCE_MODE, PROP_WHITE_POINT, PROP_ZOOM, + PROP_EXPOSURE_MODE, /*< private > */ PROP_N_INSTALL = PROP_MAX_IMAGES + 1, - PROP_LAST = PROP_ZOOM, + PROP_LAST = PROP_EXPOSURE_MODE, } GstAHC2SrcProperty; static GParamSpec *properties[PROP_LAST + 1]; @@ -486,6 +487,24 @@ gst_ahc2_src_set_focus_mode (GstPhotography * photo, return TRUE; } +static gboolean +gst_ahc2_src_get_exposure_mode (GstPhotography * photo, + GstPhotographyExposureMode * exposure_mode) +{ + (void) photo; + (void) exposure_mode; + return FALSE; +} + +static gboolean +gst_ahc2_src_set_exposure_mode (GstPhotography * photo, + GstPhotographyExposureMode exposure_mode) +{ + (void) photo; + (void) exposure_mode; + return TRUE; +} + static gboolean gst_ahc2_src_get_ev_compensation (GstPhotography * photo, gfloat * ev_comp) { @@ -1615,6 +1634,11 @@ gst_ahc2_src_set_property (GObject * object, gst_ahc2_src_set_zoom (GST_PHOTOGRAPHY (self), zoom); break; } + case PROP_EXPOSURE_MODE:{ + GstPhotographyExposureMode exposure_mode = g_value_get_enum (value); + gst_ahc2_src_set_exposure_mode (GST_PHOTOGRAPHY (self), exposure_mode); + break; + } default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1703,6 +1727,13 @@ gst_ahc2_src_get_property (GObject * object, g_value_set_float (value, zoom); break; } + case PROP_EXPOSURE_MODE:{ + GstPhotographyExposureMode exposure_mode; + if (gst_ahc2_src_get_exposure_mode (GST_PHOTOGRAPHY (self), + &exposure_mode)) + g_value_set_enum (value, exposure_mode); + break; + } default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -2138,6 +2169,11 @@ gst_ahc2_src_class_init (GstAHC2SrcClass * klass) properties[PROP_ZOOM] = g_object_class_find_property (gobject_class, GST_PHOTOGRAPHY_PROP_ZOOM); + g_object_class_override_property (gobject_class, + PROP_EXPOSURE_MODE, GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE); + properties[PROP_EXPOSURE_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE); + signals[SIG_GET_CAMERA_ID_BY_INDEX] = g_signal_new ("get-camera-id-by-index", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (GstAHC2SrcClass, @@ -2245,6 +2281,9 @@ gst_ahc2_src_photography_init (gpointer g_iface, gpointer iface_data) iface->get_focus_mode = gst_ahc2_src_get_focus_mode; iface->set_focus_mode = gst_ahc2_src_set_focus_mode; + iface->get_exposure_mode = gst_ahc2_src_get_exposure_mode; + iface->set_exposure_mode = gst_ahc2_src_set_exposure_mode; + iface->get_flash_mode = gst_ahc2_src_get_flash_mode; iface->set_flash_mode = gst_ahc2_src_set_flash_mode; From 74a4076902c05c9bc60b158296822776339441ac Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Thu, 17 Nov 2022 21:45:54 +0900 Subject: [PATCH 267/339] gstvalue/test: initialize unset string It is possible to do a comparison when the ASSERT checks are disabled --- subprojects/gstreamer/tests/check/gst/gstvalue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gstreamer/tests/check/gst/gstvalue.c b/subprojects/gstreamer/tests/check/gst/gstvalue.c index b1d2548584c..c9f055a784c 100644 --- a/subprojects/gstreamer/tests/check/gst/gstvalue.c +++ b/subprojects/gstreamer/tests/check/gst/gstvalue.c @@ -450,7 +450,7 @@ GST_END_TEST; GST_START_TEST (test_serialize_flags_invalid) { GValue value = { 0 }; - gchar *string; + gchar *string = NULL; g_value_init (&value, GST_TYPE_SEEK_FLAGS); From a29601a16ea022b007ba32ae9e3ddad21c9ba681 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 21 Nov 2022 14:03:48 +0100 Subject: [PATCH 268/339] gstv4l2deviceprovider: return the device list in the same order of appearance --- subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c index 5eeec7dc8b1..a3bbbccf934 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c @@ -218,7 +218,7 @@ gst_v4l2_device_provider_probe (GstDeviceProvider * provider) if (device) { gst_object_ref_sink (device); - devices = g_list_prepend (devices, device); + devices = g_list_append (devices, device); } } From 1eee1bde863176a1186748f810ba448952113ec4 Mon Sep 17 00:00:00 2001 From: Frederik Vestre Date: Mon, 21 Nov 2022 09:35:01 +0100 Subject: [PATCH 269/339] Media: Try to improve locking for gstvideoencoder --- .../gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c index d0efd5a1758..0d548b09c46 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c @@ -1182,11 +1182,11 @@ gst_video_encoder_sink_event_default (GstVideoEncoder * encoder, NULL, NULL, &running_time, &all_headers, &count)) { ForcedKeyUnitEvent *fevt; - GST_OBJECT_LOCK (encoder); + GST_VIDEO_ENCODER_STREAM_LOCK (encoder); fevt = forced_key_unit_event_new (running_time, all_headers, count); g_queue_insert_sorted (&encoder->priv->force_key_unit, fevt, (GCompareDataFunc) forced_key_unit_event_compare, NULL); - GST_OBJECT_UNLOCK (encoder); + GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder); GST_DEBUG_OBJECT (encoder, "force-key-unit event: running-time %" GST_TIME_FORMAT From 8a581c71001a7d626a03786b633cb9d51bdfa1f3 Mon Sep 17 00:00:00 2001 From: Frederik Vestre Date: Thu, 24 Nov 2022 13:20:09 +0100 Subject: [PATCH 270/339] More videoenc lock fixes --- .../gst-libs/gst/video/gstvideoencoder.c | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c index 0d548b09c46..78015735c74 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c @@ -458,6 +458,7 @@ gst_video_encoder_reset (GstVideoEncoder * encoder, gboolean hard) priv->presentation_frame_number = 0; priv->distance_from_sync = 0; + GST_OBJECT_LOCK (encoder); g_queue_clear_full (&priv->force_key_unit, (GDestroyNotify) forced_key_unit_event_free); priv->last_force_key_unit_request = GST_CLOCK_TIME_NONE; @@ -465,7 +466,6 @@ gst_video_encoder_reset (GstVideoEncoder * encoder, gboolean hard) priv->drained = TRUE; - GST_OBJECT_LOCK (encoder); priv->bytes = 0; priv->time = 0; GST_OBJECT_UNLOCK (encoder); @@ -1182,11 +1182,11 @@ gst_video_encoder_sink_event_default (GstVideoEncoder * encoder, NULL, NULL, &running_time, &all_headers, &count)) { ForcedKeyUnitEvent *fevt; - GST_VIDEO_ENCODER_STREAM_LOCK (encoder); + GST_OBJECT_LOCK (encoder); fevt = forced_key_unit_event_new (running_time, all_headers, count); g_queue_insert_sorted (&encoder->priv->force_key_unit, fevt, (GCompareDataFunc) forced_key_unit_event_compare, NULL); - GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder); + GST_OBJECT_UNLOCK (encoder); GST_DEBUG_OBJECT (encoder, "force-key-unit event: running-time %" GST_TIME_FORMAT @@ -2517,6 +2517,7 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, gboolean send_headers = FALSE; gboolean key_unit = FALSE; gboolean discont = FALSE; + gboolean force_key_unit_present; GstBuffer *buffer; g_return_val_if_fail (frame, GST_FLOW_ERROR); @@ -2551,7 +2552,11 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, priv->processed++; - if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && priv->force_key_unit.head) + GST_OBJECT_LOCK (encoder); + force_key_unit_present = priv->force_key_unit.head != NULL; + GST_OBJECT_UNLOCK (encoder); + + if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && force_key_unit_present) gst_video_encoder_send_key_unit_unlocked (encoder, frame, &send_headers); if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) @@ -2669,6 +2674,7 @@ gst_video_encoder_finish_subframe (GstVideoEncoder * encoder, gboolean discont = FALSE; gboolean send_headers = FALSE; gboolean key_unit = FALSE; + gboolean force_key_unit_present; g_return_val_if_fail (frame, GST_FLOW_ERROR); g_return_val_if_fail (frame->output_buffer, GST_FLOW_ERROR); @@ -2691,7 +2697,11 @@ gst_video_encoder_finish_subframe (GstVideoEncoder * encoder, if (ret != GST_FLOW_OK) goto done; - if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && priv->force_key_unit.head) + GST_OBJECT_LOCK (encoder); + force_key_unit_present = priv->force_key_unit.head != NULL; + GST_OBJECT_UNLOCK (encoder); + + if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && force_key_unit_present) gst_video_encoder_send_key_unit_unlocked (encoder, frame, &send_headers); /* Push pending events only for the first subframe ie segment event. From 40049d1f18a5486dc32031fbfb6d87a31108990c Mon Sep 17 00:00:00 2001 From: "sergei.kovalev" Date: Wed, 7 Dec 2022 15:43:24 +0100 Subject: [PATCH 271/339] gstvideorate: Fix of buffer duration. --- .../gst/videorate/gstvideorate.c | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c b/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c index ec3a30e2588..4cd8dcc5bf2 100644 --- a/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c +++ b/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c @@ -1559,15 +1559,23 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) /* we need to have two buffers to compare */ if (videorate->prevbuf == NULL || videorate->drop_only) { - /* We can calculate the duration of the buffer here if not given for - * reverse playback. We need this later */ - if (videorate->segment.rate < 0.0 && !GST_BUFFER_DURATION_IS_VALID (buffer)) { - /* As we require valid timestamps all the time for reverse playback, we either - * have a valid last_ts or we're at the very first buffer. */ - if (!GST_CLOCK_TIME_IS_VALID (last_ts)) - GST_BUFFER_DURATION (buffer) = videorate->segment.stop - in_ts; - else - GST_BUFFER_DURATION (buffer) = last_ts - in_ts; + /* We can calculate the duration of the buffer here if not given */ + if (!GST_BUFFER_DURATION_IS_VALID (buffer)) { + /* For reverse playback. We need this later */ + if (videorate->segment.rate < 0.0) { + /* As we require valid timestamps all the time for reverse playback, we either + * have a valid last_ts or we're at the very first buffer. */ + if (!GST_CLOCK_TIME_IS_VALID (last_ts)) + GST_BUFFER_DURATION (buffer) = videorate->segment.stop - in_ts; + else + GST_BUFFER_DURATION (buffer) = last_ts - in_ts; + } else if (videorate->to_rate_numerator) { + GST_BUFFER_DURATION (buffer) = gst_util_uint64_scale (1, + videorate->to_rate_denominator * GST_SECOND, + videorate->to_rate_numerator); + } else { + goto invalid_buffer_duration; + } } gst_video_rate_swap_prev (videorate, buffer, intime); @@ -1812,6 +1820,14 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) goto done; } +invalid_buffer_duration: + { + GST_WARNING_OBJECT (videorate, + "Got first buffer with GST_CLOCK_TIME_NONE duration, discarding it"); + res = GST_BASE_TRANSFORM_FLOW_DROPPED; + goto done; + } + outside_segment: { GST_WARNING_OBJECT (videorate, "Got buffer outide segment, discarding it"); @@ -1966,4 +1982,4 @@ GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, videorate, "Adjusts video frames", - plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) \ No newline at end of file + plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) From 70863929dd6aaf246d74cf45f2823127aff6e8b9 Mon Sep 17 00:00:00 2001 From: Vadym Markov Date: Fri, 25 Nov 2022 15:19:37 +0100 Subject: [PATCH 272/339] Build MoltenVK for iOS and macOS --- .../gst-libs/gst/vulkan/meson.build | 64 +++++++++++++++---- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build index 8a1e946d1e8..d8b0069a3b6 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build @@ -111,7 +111,28 @@ if ['ios', 'darwin'].contains(host_system) # retrieving the metal device from the VkDevice) which is currently waiting # on implementing a proper Metal extension for Vulkan # https://github.com/KhronosGroup/MoltenVK/issues/492 - vulkan_dep = cc.find_library('MoltenVK', required : get_option('vulkan')) + + # 1. Download and install the Vulkan SDK for macOS: + # https://vulkan.lunarg.com/sdk/home#mac + # 2. Set a shell environment variable to point to the SDK: + # `export VK_SDK_PATH="/path/to/VulkanSDK/1.3.231.1"` + vulkan_root = run_command(python3, '-c', 'import os; print(os.environ.get("VK_SDK_PATH"))', check: false).stdout().strip() + + if vulkan_root != '' and vulkan_root != 'None' + molten_vk_root = join_paths(vulkan_root, 'MoltenVK') + platform_dir = host_system == 'ios' ? 'ios-arm64' : 'macos-arm64_x86_64' + vulkan_lib_dir = join_paths(molten_vk_root, 'MoltenVK.xcframework', platform_dir) + vulkan_inc_dir = join_paths(molten_vk_root, 'include') + vulkan_lib = cc.find_library('MoltenVK', dirs: vulkan_lib_dir, + required : get_option('vulkan')) + has_vulkan_header = cc.has_header('vulkan/vulkan_core.h', + include_directories: include_directories(vulkan_inc_dir)) + vulkan_dep = declare_dependency(include_directories: include_directories(vulkan_inc_dir), + dependencies: vulkan_lib, + link_args: ['-lc++']) + else + subdir_done() + endif elif host_system == 'windows' vulkan_root = run_command(python3, '-c', 'import os; print(os.environ.get("VK_SDK_PATH"))', check: false).stdout().strip() if vulkan_root != '' and vulkan_root != 'None' @@ -141,7 +162,7 @@ else endif endif -if host_system != 'windows' +if not ['ios', 'darwin', 'windows'].contains(host_system) has_vulkan_header = cc.has_header('vulkan/vulkan_core.h') endif @@ -224,41 +245,60 @@ if ['darwin', 'ios'].contains(host_system) vulkan_objc_args += ['-fobjc-arc'] - foundation_dep = dependency('appleframeworks', modules : ['Foundation'], required : get_option('vulkan')) - quartzcore_dep = dependency('appleframeworks', modules : ['QuartzCore'], required : get_option('vulkan')) - corefoundation_dep = dependency('appleframeworks', modules : ['CoreFoundation'], required : get_option('vulkan')) - if foundation_dep.found() and quartzcore_dep.found() and corefoundation_dep.found() - optional_deps += [foundation_dep, corefoundation_dep, quartzcore_dep] - endif + apple_deps = [ + dependency('appleframeworks', modules : ['Foundation'], required : get_option('vulkan')), + dependency('appleframeworks', modules : ['QuartzCore'], required : get_option('vulkan')), + dependency('appleframeworks', modules : ['CoreFoundation'], required : get_option('vulkan')), + dependency('appleframeworks', modules : ['Metal'], required : get_option('vulkan')), + dependency('appleframeworks', modules : ['IOSurface'], required : get_option('vulkan')), + ] + + foreach dep : apple_deps + if dep.found() + optional_deps += [dep] + endif + endforeach endif if host_system == 'darwin' cocoa_dep = dependency('appleframeworks', modules : ['Cocoa'], required : get_option('vulkan')) - if cocoa_dep.found() and cc.has_header('vulkan/vulkan_macos.h', dependencies : vulkan_dep) + if cocoa_dep.found() and cc.has_header('vulkan/vulkan_macos.h', include_directories: include_directories(vulkan_inc_dir)) vulkan_priv_sources += files( 'cocoa/gstvkdisplay_cocoa.m', 'cocoa/gstvkwindow_cocoa.m', ) - optional_deps += [cocoa_dep] vulkan_windowing = true vulkan_conf.set('GST_VULKAN_HAVE_WINDOW_COCOA', 1) enabled_vulkan_winsys += ['cocoa'] + + iokit_dep = dependency('appleframeworks', modules : ['IOKit'], required : get_option('vulkan')) + if cocoa_dep.found() + optional_deps += [iokit_dep] + endif + + optional_deps += [cocoa_dep] endif endif if host_system == 'ios' uikit_dep = dependency('appleframeworks', modules : ['UIKit'], required : get_option('vulkan')) - if uikit_dep.found() and cc.has_header('vulkan/vulkan_ios.h', dependencies : vulkan_dep) + if uikit_dep.found() and cc.has_header('vulkan/vulkan_ios.h', include_directories: include_directories(vulkan_inc_dir)) vulkan_priv_sources += files( 'ios/gstvkdisplay_ios.m', 'ios/gstvkwindow_ios.m', ) - optional_deps += [uikit_dep] vulkan_windowing = true vulkan_conf.set('GST_VULKAN_HAVE_WINDOW_IOS', 1) enabled_vulkan_winsys += ['ios'] + + coregraphics_dep = dependency('appleframeworks', modules : ['CoreGraphics'], required : get_option('vulkan')) + if coregraphics_dep.found() + optional_deps += [coregraphics_dep] + endif + + optional_deps += [uikit_dep] endif endif From 6a3e000451c4676e05c4d2c46599c90c0960a305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Fri, 16 Dec 2022 13:52:56 +0100 Subject: [PATCH 273/339] video-info: PEXHACK reduce warning levels for spammy log --- subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c b/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c index d140dfabb06..948337eb3e9 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c @@ -197,7 +197,7 @@ validate_colorimetry (GstVideoInfo * info) if (GST_VIDEO_FORMAT_INFO_IS_YUV (finfo) && info->colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_UNKNOWN) { - GST_WARNING ("Need to specify a color matrix when using YUV format (%s)", + GST_INFO ("Need to specify a color matrix when using YUV format (%s)", finfo->name); return FALSE; } @@ -513,7 +513,7 @@ gst_video_info_from_caps (GstVideoInfo * info, const GstCaps * caps) GST_WARNING ("unparsable colorimetry, using default"); set_default_colorimetry (info); } else if (!validate_colorimetry (info)) { - GST_WARNING ("invalid colorimetry, using default"); + GST_INFO ("invalid colorimetry, using default"); set_default_colorimetry (info); } else { /* force RGB matrix for RGB formats */ From c690b9280d055cea96fb0c853685211313d6e1e8 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 9 Dec 2022 15:34:51 +0100 Subject: [PATCH 274/339] multiudpsink: use "current-time" not "running-time" for TWCC --- subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c b/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c index f13e35b7b31..f5504c37774 100644 --- a/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c +++ b/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c @@ -719,14 +719,13 @@ _set_time_on_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, guint num_buffers) { GstClock *clock = GST_ELEMENT_CLOCK (sink); - GstClockTime base_time = GST_ELEMENT_CAST (sink)->base_time; GstClockTime now; guint i; if (clock == NULL) return; - now = gst_clock_get_time (clock) - base_time; + now = gst_clock_get_time (clock); for (i = 0; i < num_buffers; i++) { GstTxFeedbackMeta *meta = gst_buffer_get_tx_feedback_meta (buffers[i]); From 96653f3b4da85d3bb8651143845656406ed69af1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Mon, 23 Jan 2023 23:23:00 +0100 Subject: [PATCH 275/339] Revert "gstpad: PEXHACK: log chain(list)funcs that take longer than 100ms" This reverts commit 94340831a7b7a889fb5b3793d6936c26aed0a687. --- subprojects/gstreamer/gst/gstpad.c | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/subprojects/gstreamer/gst/gstpad.c b/subprojects/gstreamer/gst/gstpad.c index 93c2a810872..515c28fcd80 100644 --- a/subprojects/gstreamer/gst/gstpad.c +++ b/subprojects/gstreamer/gst/gstpad.c @@ -4429,10 +4429,6 @@ gst_pad_chain_data_unchecked (GstPad * pad, GstPadProbeType type, void *data) GST_TRACER_PAD_CHAIN_PRE (pad, data); } -/* FIXME: PEXHACK BEGIN */ - gint64 start, dur; -/* FIXME: PEXHACK END */ - GST_PAD_STREAM_LOCK (pad); GST_OBJECT_LOCK (pad); @@ -4484,20 +4480,8 @@ gst_pad_chain_data_unchecked (GstPad * pad, GstPadProbeType type, void *data) "calling chainfunction &%s with buffer %" GST_PTR_FORMAT, GST_DEBUG_FUNCPTR_NAME (chainfunc), GST_BUFFER (data)); -/* FIXME: PEXHACK BEGIN */ - start = g_get_monotonic_time (); -/* FIXME: PEXHACK END */ - ret = chainfunc (pad, parent, GST_BUFFER_CAST (data)); -/* FIXME: PEXHACK BEGIN */ - dur = (g_get_monotonic_time () - start) / G_TIME_SPAN_MILLISECOND; - if (dur >= 100) { - GST_WARNING_OBJECT (pad, "Slow chainfunc (%s) %" G_GINT64_FORMAT "ms", - GST_DEBUG_FUNCPTR_NAME (chainfunc), dur); - } -/* FIXME: PEXHACK END */ - GST_CAT_DEBUG_OBJECT (GST_CAT_SCHEDULING, pad, "called chainfunction &%s with buffer %p, returned %s", GST_DEBUG_FUNCPTR_NAME (chainfunc), data, gst_flow_get_name (ret)); @@ -4511,20 +4495,8 @@ gst_pad_chain_data_unchecked (GstPad * pad, GstPadProbeType type, void *data) "calling chainlistfunction &%s", GST_DEBUG_FUNCPTR_NAME (chainlistfunc)); -/* FIXME: PEXHACK BEGIN */ - start = g_get_monotonic_time (); -/* FIXME: PEXHACK END */ - ret = chainlistfunc (pad, parent, GST_BUFFER_LIST_CAST (data)); -/* FIXME: PEXHACK BEGIN */ - dur = (g_get_monotonic_time () - start) / G_TIME_SPAN_MILLISECOND; - if (dur >= 100) { - GST_WARNING_OBJECT (pad, "Slow chainlistfunc (%s) %" G_GINT64_FORMAT "ms", - GST_DEBUG_FUNCPTR_NAME (chainlistfunc), dur); - } -/* FIXME: PEXHACK END */ - GST_CAT_DEBUG_OBJECT (GST_CAT_SCHEDULING, pad, "called chainlistfunction &%s, returned %s", GST_DEBUG_FUNCPTR_NAME (chainlistfunc), gst_flow_get_name (ret)); From 8cb6ec86158e9177830f90560cad6dab7b4c0563 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 6 Feb 2023 12:53:49 +0100 Subject: [PATCH 276/339] gst-plugins-base/playback: scanbuild fixes --- .../gst/playback/gstdecodebin2.c | 2 +- .../gst-plugins-base/gst/playback/gstplaybin2.c | 4 ++-- .../gst/playback/gsturidecodebin.c | 10 +++++----- .../gst/playback/gsturisourcebin.c | 16 ++++++++-------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/subprojects/gst-plugins-base/gst/playback/gstdecodebin2.c b/subprojects/gst-plugins-base/gst/playback/gstdecodebin2.c index 2e8c943118a..4a8116481a8 100644 --- a/subprojects/gst-plugins-base/gst/playback/gstdecodebin2.c +++ b/subprojects/gst-plugins-base/gst/playback/gstdecodebin2.c @@ -3342,7 +3342,7 @@ gst_decode_chain_get_current_group (GstDecodeChain * chain) for (iter = chain->next_groups; iter; iter = g_list_next (iter)) { GstDecodeGroup *next_group = iter->data; - if (!next_group->overrun && !next_group->no_more_pads) { + if (next_group && !next_group->overrun && !next_group->no_more_pads) { group = next_group; break; } diff --git a/subprojects/gst-plugins-base/gst/playback/gstplaybin2.c b/subprojects/gst-plugins-base/gst/playback/gstplaybin2.c index 5a4ca4404c3..e1b8b9bf144 100644 --- a/subprojects/gst-plugins-base/gst/playback/gstplaybin2.c +++ b/subprojects/gst-plugins-base/gst/playback/gstplaybin2.c @@ -3024,11 +3024,11 @@ gst_play_bin_handle_message (GstBin * bin, GstMessage * msg) if (gst_uri_is_valid (location)) { uri = g_strdup (location); - } else { + } else if (group) { uri = gst_uri_join_strings (group->uri, location); } - if (g_strcmp0 (uri, group->uri)) { + if (uri && group && g_strcmp0 (uri, group->uri)) { GST_PLAY_BIN_LOCK (playbin); if (playbin->next_group && playbin->next_group->valid) { GST_DEBUG_OBJECT (playbin, diff --git a/subprojects/gst-plugins-base/gst/playback/gsturidecodebin.c b/subprojects/gst-plugins-base/gst/playback/gsturidecodebin.c index 27ec4f3c40e..67fa8a491e6 100644 --- a/subprojects/gst-plugins-base/gst/playback/gsturidecodebin.c +++ b/subprojects/gst-plugins-base/gst/playback/gsturidecodebin.c @@ -1305,11 +1305,11 @@ array_has_value (const gchar * values[], const gchar * value) } static gboolean -array_has_uri_value (const gchar * values[], const gchar * value) +array_has_uri_value (const gchar * values[], gint nelems, const gchar * value) { gint i; - for (i = 0; values[i]; i++) { + for (i = 0; i < nelems && values[i]; i++) { if (!g_ascii_strncasecmp (value, values[i], strlen (values[i]))) return TRUE; } @@ -1336,9 +1336,9 @@ static const gchar *adaptive_media[] = { "application/dash+xml", NULL }; -#define IS_STREAM_URI(uri) (array_has_uri_value (stream_uris, uri)) -#define IS_QUEUE_URI(uri) (array_has_uri_value (queue_uris, uri)) -#define IS_BLACKLISTED_URI(uri) (array_has_uri_value (blacklisted_uris, uri)) +#define IS_STREAM_URI(uri) (array_has_uri_value (stream_uris, G_N_ELEMENTS(stream_uris), uri)) +#define IS_QUEUE_URI(uri) (array_has_uri_value (queue_uris, G_N_ELEMENTS(queue_uris), uri)) +#define IS_BLACKLISTED_URI(uri) (array_has_uri_value (blacklisted_uris, G_N_ELEMENTS(blacklisted_uris), uri)) #define IS_ADAPTIVE_MEDIA(media) (array_has_value (adaptive_media, media)) /* diff --git a/subprojects/gst-plugins-base/gst/playback/gsturisourcebin.c b/subprojects/gst-plugins-base/gst/playback/gsturisourcebin.c index 8dcf371bdfd..ae5db1d8100 100644 --- a/subprojects/gst-plugins-base/gst/playback/gsturisourcebin.c +++ b/subprojects/gst-plugins-base/gst/playback/gsturisourcebin.c @@ -1443,11 +1443,11 @@ demuxer_pad_removed_cb (GstElement * element, GstPad * pad, /* helper function to lookup stuff in lists */ static gboolean -array_has_value (const gchar * values[], const gchar * value) +array_has_value (const gchar * values[], gint nelems, const gchar * value) { gint i; - for (i = 0; values[i]; i++) { + for (i = 0; i < nelems && values[i]; i++) { if (g_str_has_prefix (value, values[i])) return TRUE; } @@ -1455,11 +1455,11 @@ array_has_value (const gchar * values[], const gchar * value) } static gboolean -array_has_uri_value (const gchar * values[], const gchar * value) +array_has_uri_value (const gchar * values[], gint nelems, const gchar * value) { gint i; - for (i = 0; values[i]; i++) { + for (i = 0; i < nelems && values[i]; i++) { if (!g_ascii_strncasecmp (value, values[i], strlen (values[i]))) return TRUE; } @@ -1486,10 +1486,10 @@ static const gchar *adaptive_media[] = { "application/dash+xml", NULL }; -#define IS_STREAM_URI(uri) (array_has_uri_value (stream_uris, uri)) -#define IS_QUEUE_URI(uri) (array_has_uri_value (queue_uris, uri)) -#define IS_BLACKLISTED_URI(uri) (array_has_uri_value (blacklisted_uris, uri)) -#define IS_ADAPTIVE_MEDIA(media) (array_has_value (adaptive_media, media)) +#define IS_STREAM_URI(uri) (array_has_uri_value (stream_uris, G_N_ELEMENTS(stream_uris), uri)) +#define IS_QUEUE_URI(uri) (array_has_uri_value (queue_uris, G_N_ELEMENTS(queue_uris), uri)) +#define IS_BLACKLISTED_URI(uri) (array_has_uri_value (blacklisted_uris, G_N_ELEMENTS(blacklisted_uris), uri)) +#define IS_ADAPTIVE_MEDIA(media) (array_has_value (adaptive_media, G_N_ELEMENTS(adaptive_media), media)) /* * Generate and configure a source element. From 896f3778b7eb97fbed1a1b6252230f15d9efdb8f Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 14 Feb 2023 16:42:51 +0100 Subject: [PATCH 277/339] rtprtxsend: introduce stuffing-max-burst-packets and use padding In order to limit the amount of packets that is used for stuffing, we both introduce a hard limit of how many packets a "burst" can consist of, but also use padding to make packets slightly bigger, and hence reduce the need to send quite so many. --- .../gst/rtpmanager/gstrtprtxsend.c | 188 +++++++++++------- .../gst/rtpmanager/gstrtprtxsend.h | 1 + .../tests/check/elements/rtprtx.c | 78 ++++++-- 3 files changed, 181 insertions(+), 86 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c index 02e50f8aa16..55be31cae31 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c @@ -45,6 +45,7 @@ #include #include "gstrtprtxsend.h" +#include "rtpstats.h" GST_DEBUG_CATEGORY_STATIC (gst_rtp_rtx_send_debug); #define GST_CAT_DEFAULT gst_rtp_rtx_send_debug @@ -59,6 +60,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_rtx_send_debug); #define DEFAULT_MAX_KBPS UNLIMITED_KBPS #define DEFAULT_MAX_BUCKET_SIZE UNLIMITED_KBPS #define DEFAULT_STUFFING_KBPS 0 +#define DEFAULT_STUFFING_MAX_BURST_PACKETS 20 enum { @@ -73,6 +75,7 @@ enum PROP_MAX_KBPS, PROP_MAX_BUCKET_SIZE, PROP_STUFFING_KBPS, + PROP_STUFFING_MAX_BURST_PACKETS, PROP_LAST, }; @@ -218,6 +221,12 @@ ssrc_rtx_data_free (SSRCRtxData * data) g_free (data); } +static gint +get_buffer_bytes_size (GstBuffer * buffer) +{ + return gst_buffer_get_size (buffer) + UDP_IP_HEADER_OVERHEAD; +} + typedef enum { RTX_TASK_START, @@ -395,6 +404,21 @@ gst_rtp_rtx_send_class_init (GstRtpRtxSendClass * klass) 0, G_MAXINT, DEFAULT_STUFFING_KBPS, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + /** + * rtprtxsend:stuffing-max-burst-packets: + * + * The maximum amount of stuffing-packets to produce in a single burst. + * + * Since: 1.22 + */ + g_object_class_install_property (gobject_class, + PROP_STUFFING_MAX_BURST_PACKETS, + g_param_spec_int ("stuffing-max-burst-packets", + "Stuffing Max Burst Packets", + "The maximum amount of stuffing-packets to produce in a single burst", + 0, G_MAXINT, DEFAULT_STUFFING_MAX_BURST_PACKETS, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + gst_element_class_add_static_pad_template (gstelement_class, &src_factory); gst_element_class_add_static_pad_template (gstelement_class, &sink_factory); @@ -843,7 +867,7 @@ gst_rtp_rtx_send_token_bucket (GstRtpRtxSend * rtx, GstBuffer * buf) return TRUE; } - tokens = gst_buffer_get_size (buf) * 8; + tokens = get_buffer_bytes_size (buf) * 8; token_bucket_add_tokens (&rtx->max_tb, gst_clock_get_time (clock)); return token_bucket_take_tokens (&rtx->max_tb, tokens, FALSE); } @@ -1165,126 +1189,143 @@ process_buffer (GstRtpRtxSend * rtx, GstBuffer * buffer) return data; } -static gint -get_buffer_payload_len (GstBuffer * buffer) -{ - gint length = gst_buffer_get_size (buffer) - MIN_RTP_HEADER_LEN; - - if (GST_BUFFER_FLAG_IS_SET (buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION)) { - length -= RTX_OVERHEAD; - } - - return length; -} - static GstFlowReturn gst_rtp_rtx_send_push (GstRtpRtxSend * rtx, GstBuffer * buffer) { GstFlowReturn ret; - token_bucket_take_tokens (&rtx->stuff_tb, - get_buffer_payload_len (buffer) * 8, TRUE); + guint buffer_size = get_buffer_bytes_size (buffer); + token_bucket_take_tokens (&rtx->stuff_tb, buffer_size * 8, TRUE); GST_OBJECT_UNLOCK (rtx); + GST_LOG_OBJECT (rtx, + "Pushing buffer %" GST_PTR_FORMAT " with network size: %u", buffer, + buffer_size); ret = gst_pad_push (rtx->srcpad, buffer); GST_OBJECT_LOCK (rtx); return ret; } -static GSequenceIter * -gst_rtp_rtx_send_get_initial_stuffing_buffer (GstRtpRtxSend * rtx, - SSRCRtxData * rtx_data, gboolean check_window) +static guint +gst_rtp_rtx_send_get_stuffing_buffers (GstRtpRtxSend * rtx, + SSRCRtxData * rtx_data, GSequenceIter ** start, GSequenceIter ** end) { GstClockTime running_time; GstClockTime window_size = 100 * GST_MSECOND; GSequenceIter *first = NULL; GSequenceIter *last = NULL; + GSequenceIter *current; - gint available_stuffing_bits = 0; - gint64 bucket_size = rtx->stuff_tb.bucket_size; + guint available_stuffing_bytes = 0; + guint bucket_size_bytes = (guint) (rtx->stuff_tb.bucket_size / 8); + + if (g_sequence_is_empty (rtx_data->queue)) + return 0; /* determine the first and last item on the queue to do stuffing with */ + first = g_sequence_get_begin_iter (rtx_data->queue); last = g_sequence_get_end_iter (rtx_data->queue); - first = last; running_time = gst_clock_get_time (GST_ELEMENT_CLOCK (rtx)) - GST_ELEMENT_CAST (rtx)->base_time; - do { - BufferQueueItem *item = g_sequence_get (g_sequence_iter_prev (first)); - gint buffer_bitsize = get_buffer_payload_len (item->buffer) * 8; + current = last; + while (current != first) { + BufferQueueItem *item = g_sequence_get (g_sequence_iter_prev (current)); + /* the additional 2 bytes here is when turning this into a RTX buffer */ + guint buffer_size = get_buffer_bytes_size (item->buffer) + 2; + GST_LOG_OBJECT (rtx, "Considering buffer #%u with size %u, total: %u", + item->seqnum, buffer_size, available_stuffing_bytes); /* stop here if we will exceed the bucket with this buffer */ - if (available_stuffing_bits + buffer_bitsize > bucket_size) + if (available_stuffing_bytes + buffer_size > bucket_size_bytes) break; - /* stop here if the packet is "too old" */ - if (check_window && GST_BUFFER_PTS_IS_VALID (item->buffer) - && ABS (GST_CLOCK_DIFF (GST_BUFFER_PTS (item->buffer), - running_time)) > window_size) { + /* stop here if the packet is "too old", but only if we already have a packet to push */ + if (current != last && GST_BUFFER_PTS_IS_VALID (item->buffer) && + GST_CLOCK_DIFF (GST_BUFFER_PTS (item->buffer), + running_time) > window_size) { break; } - available_stuffing_bits += buffer_bitsize; - first = g_sequence_iter_prev (first); - } while (first != g_sequence_get_begin_iter (rtx_data->queue)); + current = g_sequence_iter_prev (current); + available_stuffing_bytes += buffer_size; + } - if (available_stuffing_bits) - return first; + /* we were unable to find any buffers */ + if (current == last) + return 0; - /* not enough bits for stuffing */ - return NULL; -} + *start = current; + *end = last; + return available_stuffing_bytes * 8; +} static GstFlowReturn gst_rtp_rtx_send_push_stuffing (GstRtpRtxSend * rtx, SSRCRtxData * rtx_data) { GstFlowReturn ret = GST_FLOW_OK; - GSequenceIter *first = NULL; GSequenceIter *last = NULL; + GSequenceIter *current; + guint available_stuffing_bits; + gint missing_stuffing_bytes; + guint stuffing_pushed = 0; - gint64 bucket_size = rtx->stuff_tb.bucket_size; - - if (bucket_size <= 0) - return GST_FLOW_OK; - - /* double check the queue */ - if (g_sequence_is_empty (rtx_data->queue)) + if (rtx->stuff_tb.bucket_size <= 0) { + GST_DEBUG_OBJECT (rtx, "No room in bucket for stuffing"); return GST_FLOW_OK; - - last = g_sequence_get_end_iter (rtx_data->queue); - - /* check for the first buffer under the window */ - first = gst_rtp_rtx_send_get_initial_stuffing_buffer (rtx, rtx_data, TRUE); - if (!first) { - /* if there is none, fallback to use the last ones */ - first = gst_rtp_rtx_send_get_initial_stuffing_buffer (rtx, rtx_data, FALSE); } + available_stuffing_bits = + gst_rtp_rtx_send_get_stuffing_buffers (rtx, rtx_data, &first, &last); + /* we cannot generate any stuffing, no bits */ - if (!first) + if (!available_stuffing_bits) { + GST_DEBUG_OBJECT (rtx, "No available RTX packets to use for stuffing"); return GST_FLOW_OK; + } - while (first != last) { - BufferQueueItem *item = g_sequence_get (first); - GstBuffer *rtx_buf = gst_rtp_rtx_buffer_new (rtx, item->buffer, 0); - - GST_DEBUG_OBJECT (rtx, "Produce 1 stuffing packets with ssrc %X, " - "original buffer size %" G_GSIZE_FORMAT, rtx_data->rtx_ssrc, - gst_buffer_get_size (item->buffer)); - + GST_DEBUG_OBJECT (rtx, + "About to produce up to %u bytes of stuffing using %u bytes of packet data", + (guint) (rtx->stuff_tb.bucket_size / 8), available_stuffing_bits / 8); + + missing_stuffing_bytes = + MAX (0, (rtx->stuff_tb.bucket_size - (gint) available_stuffing_bits) / 8); + + current = first; + while (ret == GST_FLOW_OK && (rtx->stuff_tb.bucket_size / 8) > 0 + && stuffing_pushed < rtx->stuffing_max_burst_packets) { + GstBuffer *rtx_buf; + BufferQueueItem *item = g_sequence_get (current); + gsize bufsize = get_buffer_bytes_size (item->buffer) + 2; + guint8 padding = MIN (G_MAXUINT8, missing_stuffing_bytes); + if (padding && bufsize < 300) { /* we only pad buffers less than 300 bytes in size */ + missing_stuffing_bytes -= padding; + GST_LOG_OBJECT (rtx, "adding %u bytes of padding, still missing: %u", + padding, missing_stuffing_bytes); + } + rtx_buf = gst_rtp_rtx_buffer_new (rtx, item->buffer, padding); ret = gst_rtp_rtx_send_push (rtx, rtx_buf); - first = g_sequence_iter_next (first); - } - bucket_size = rtx->stuff_tb.bucket_size; - /* we still have some left over budget, call this once more */ - if (bucket_size > 0 && ret == GST_FLOW_OK) - return gst_rtp_rtx_send_push_stuffing (rtx, rtx_data); + GST_LOG_OBJECT (rtx, + "Pushed 1 stuffing packet with size %u - bucket_size=%d", + (guint) get_buffer_bytes_size (rtx_buf), + (gint) (rtx->stuff_tb.bucket_size / 8)); + current = g_sequence_iter_next (current); + + /* if we are at the end, start over */ + if (current == last) { + current = first; + missing_stuffing_bytes = + MAX (0, + (rtx->stuff_tb.bucket_size - (gint) available_stuffing_bits) / 8); + } + stuffing_pushed++; + } return ret; } @@ -1480,6 +1521,11 @@ gst_rtp_rtx_send_get_property (GObject * object, g_value_set_int (value, rtx->stuffing_kbps); GST_OBJECT_UNLOCK (rtx); break; + case PROP_STUFFING_MAX_BURST_PACKETS: + GST_OBJECT_LOCK (rtx); + g_value_set_int (value, rtx->stuffing_max_burst_packets); + GST_OBJECT_UNLOCK (rtx); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1528,6 +1574,7 @@ gst_rtp_rtx_send_reset_max_bucket (GstRtpRtxSend * rtx, static void gst_rtp_rtx_send_reset_stuffing_bucket (GstRtpRtxSend * rtx, gint kbps) { + GST_INFO_OBJECT (rtx, "Setting stuffing to %u kbps", kbps); rtx->stuffing_kbps = kbps; token_bucket_init (&rtx->stuff_tb, kbps * 1000, kbps * 1000); } @@ -1606,6 +1653,11 @@ gst_rtp_rtx_send_set_property (GObject * object, } GST_OBJECT_UNLOCK (rtx); break; + case PROP_STUFFING_MAX_BURST_PACKETS: + GST_OBJECT_LOCK (rtx); + rtx->stuffing_max_burst_packets = g_value_get_int (value); + GST_OBJECT_UNLOCK (rtx); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h index 42a24719615..717d8be2ef0 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h @@ -92,6 +92,7 @@ struct _GstRtpRtxSend /* stuffing properties */ gint stuffing_kbps; + guint stuffing_max_burst_packets; /* last ssrc used for stuffing */ gint last_stuffing_ssrc; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c index d3b77d64153..890affb1773 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c @@ -1440,25 +1440,19 @@ GST_START_TEST (test_rtxsender_stuffing) gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 600)); pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); - /* budget 2000, sent 1200, stuff with #2, 600 bytes */ pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); - /* budget 2000, sent 1800, no stuffing */ gst_harness_set_time (h, 30 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 3, 200)); pull_and_verify (h, FALSE, master_ssrc, master_pt, 3); - /* budget 3000, sent 2000. Stuffed packets will be of (600+200+200)=1000 bytes */ - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); - /* budget 3000, sent 3000 */ gst_harness_set_time (h, 40 * GST_MSECOND); gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 4, 600)); pull_and_verify (h, FALSE, master_ssrc, master_pt, 4); - /* budget 4000, sent 3600, no stuffing because rtx packet is 600 */ g_usleep (G_USEC_PER_SEC / 100); fail_if (gst_harness_try_pull (h)); @@ -1511,7 +1505,7 @@ GST_START_TEST (test_rtxsender_stuffing_toggle) /* 80 kbps = 10 kBps. 10 ms distance between each packet means 10 packets * per second and 100 bytes per packet to hit target kbps. * We stuff considering the payload length. */ - g_object_set (h->element, "stuffing-kbps", 80, NULL); + g_object_set (h->element, "stuffing-kbps", 245, NULL); gst_harness_set_time (h, 30 * GST_MSECOND); gst_harness_push (h, @@ -1627,7 +1621,6 @@ GST_START_TEST (test_rtxsender_stuffing_window) GstStructure *pt_map, *ssrc_map; GstHarness *h = gst_harness_new ("rtprtxsend"); GstBuffer *buf; - gint i; pt_map = gst_structure_new ("application/x-rtp-pt-map", "96", G_TYPE_UINT, rtx_pt, NULL); @@ -1666,10 +1659,8 @@ GST_START_TEST (test_rtxsender_stuffing_window) pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); /* we have budget, so stuff with #1 and #2, but #0 is too old */ - for (i = 0; i < 5; i++) { - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); - } + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); g_usleep (G_USEC_PER_SEC / 100); fail_if (gst_harness_try_pull (h)); @@ -1731,10 +1722,9 @@ GST_START_TEST (test_rtxsender_stuffing_no_packets_under_window) gst_harness_push (h, buf); pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); - /* we have budget, non of the above are under the window, but we get stuff with: #0, 1 and 2 */ - for (i = 0; i < 5; i++) { - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 0); - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + /* we have budget, non of the above are under the window, so we stuff + repeatedly with the latest buffer */ + for (i = 0; i < 4; i++) { pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); } @@ -1830,8 +1820,56 @@ GST_START_TEST (test_rtxsender_stuffing_does_not_interfer_with_rtx) gst_harness_push (h, create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 400)); pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); - /* budet 2000, sent 1000, send #2: 402 bytes of stuffing */ - pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +static guint MAX_BURST_PACKETS[] = { + 5, 10, 100, 500, 777, +}; + +GST_START_TEST (test_rtxsender_stuffing_max_burst_packets) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + guint max_burst_packets = MAX_BURST_PACKETS[__i__]; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, + "stuffing-kbps", 2000, + "stuffing-max-burst-packets", max_burst_packets, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (h, 0 * GST_MSECOND); + /* push only a very small buffer to force the creation of a lot of stuffing */ + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + /* set the time 1 second in the future, to create a huge deficit on stuffing */ + gst_harness_set_time (h, 1 * GST_SECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); + + /* verify we don't burst out more than the configured max-burst, plus the + one packet we pushed */ + fail_unless_equals_int (gst_harness_buffers_in_queue (h), + max_burst_packets + 1); gst_structure_free (pt_map); gst_structure_free (ssrc_map); @@ -1879,6 +1917,10 @@ rtprtx_suite (void) test_rtxsender_stuffing_sanity_when_input_rate_is_extreme); tcase_add_test (tc_chain, test_rtxsender_stuffing_does_not_interfer_with_rtx); + tcase_add_loop_test (tc_chain, test_rtxsender_stuffing_max_burst_packets, + 0, G_N_ELEMENTS (MAX_BURST_PACKETS)); + + return s; } From a401ca2f445a930adec4f10cf52c963f601fede9 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 28 Feb 2023 13:16:19 +0100 Subject: [PATCH 278/339] funnel: only forward events on active pads IF we are going to forward events on non-active pads, we need to send our sticky events prior to this event to make it valid. However, this will lead to the situation of getting a caps change downstream for any event being received on either pad in the funnel. In the case of a decoder sitting downsstream, this would mean potentially replacing that decoder on a caps-change, meaning an "irrelevant" event received on a non-active pad would cause a decoder to lose all state, potentially needing an I-frame or similar. --- .../gstreamer/plugins/elements/gstfunnel.c | 32 +- .../gstreamer/tests/check/elements/funnel.c | 323 +++++++++++------- 2 files changed, 194 insertions(+), 161 deletions(-) diff --git a/subprojects/gstreamer/plugins/elements/gstfunnel.c b/subprojects/gstreamer/plugins/elements/gstfunnel.c index 9be847fc5b6..4575cc63996 100644 --- a/subprojects/gstreamer/plugins/elements/gstfunnel.c +++ b/subprojects/gstreamer/plugins/elements/gstfunnel.c @@ -361,16 +361,6 @@ gst_funnel_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GST_MINI_OBJECT_CAST (buffer)); } -static gboolean -_is_lost_event (GstEvent * event) -{ - const GstStructure *s; - s = gst_event_get_structure (event); - if (s) - return gst_structure_has_name (s, "GstRTPPacketLost"); - return FALSE; -} - static gboolean gst_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -379,7 +369,6 @@ gst_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) gboolean forward = TRUE; gboolean res = TRUE; gboolean unlock = FALSE; - gboolean is_lost_event; GST_DEBUG_OBJECT (pad, "received event %" GST_PTR_FORMAT, event); @@ -405,25 +394,8 @@ gst_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) GST_OBJECT_LOCK (funnel); fpad->got_eos = FALSE; GST_OBJECT_UNLOCK (funnel); - } - - /* FIXME: make something better around lost-events. rtpfunnel? */ - is_lost_event = _is_lost_event (event); - if (!is_lost_event && forward && GST_EVENT_IS_SERIALIZED (event)) { - /* If no data is coming and we receive serialized event, need to forward all sticky events. - * Otherwise downstream has an inconsistent set of sticky events when - * handling the new event. */ - if (!unlock) { - unlock = TRUE; - GST_PAD_STREAM_LOCK (funnel->srcpad); - } - - if ((funnel->last_sinkpad == NULL) || (funnel->forward_sticky_events - && (funnel->last_sinkpad != pad))) { - gst_object_replace ((GstObject **) & funnel->last_sinkpad, - GST_OBJECT (pad)); - gst_pad_sticky_events_foreach (pad, forward_events, funnel->srcpad); - } + } else if (pad != funnel->last_sinkpad) { + forward = FALSE; } if (forward) diff --git a/subprojects/gstreamer/tests/check/elements/funnel.c b/subprojects/gstreamer/tests/check/elements/funnel.c index d88628e4316..b9efa92a3bb 100644 --- a/subprojects/gstreamer/tests/check/elements/funnel.c +++ b/subprojects/gstreamer/tests/check/elements/funnel.c @@ -108,7 +108,8 @@ static gint bufcount = 0; static gint alloccount = 0; static GstFlowReturn -chain_ok (GstPad * pad, GstObject * parent, GstBuffer * buffer) +chain_ok (G_GNUC_UNUSED GstPad * pad, G_GNUC_UNUSED GstObject * parent, + GstBuffer * buffer) { bufcount++; @@ -244,135 +245,6 @@ GST_START_TEST (test_funnel_eos) GST_END_TEST; -guint nb_stream_start_event = 0; -guint nb_caps_event = 0; -guint nb_segment_event = 0; -guint nb_gap_event = 0; - -static GstPadProbeReturn -event_counter (GstObject * pad, GstPadProbeInfo * info, gpointer user_data) -{ - GstEvent *event = GST_PAD_PROBE_INFO_EVENT (info); - - fail_unless (event != NULL); - fail_unless (GST_IS_EVENT (event)); - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_STREAM_START: - ++nb_stream_start_event; - break; - case GST_EVENT_CAPS: - ++nb_caps_event; - break; - case GST_EVENT_SEGMENT: - ++nb_segment_event; - break; - case GST_EVENT_GAP: - ++nb_gap_event; - break; - default: - break; - } - - return GST_PAD_PROBE_OK; -} - -/* - * Push GAP events into funnel to forward sticky events. - * Funnel element should also treat GAP events likes buffers. - * For example, funnel can be used for internal subtitle with streamiddemux. - * +--------------------------------------------------------------------------+ - * | playbin +--------------------------------+ | - * | +--------------+ +----------------+ | +------------+ playsink | | - * | | uridecodebin | | input-selector | | | video-sink | | | - * | | | +----------------+ | +------------+ | | - * | | | | | | - * | | | +----------------+ | +------------+ | | - * | | | | input-selector | | | audio-sink | | | - * | | | +----------------+ | +------------+ | | - * | | | | | | - * | | | +----------------+ | +---------------+ +----------+ | | - * | | | | funnel | | | streamiddemux | | appsink0 | | | - * | +--------------+ +----------------+ | +---------------+ +----------+ | | - * | | +----------+ | | - * | | | appsinkn | | | - * | | +----------+ | | - * | +--------------------------------+ | - * +--------------------------------------------------------------------------+ - * If no data was received in funnel and then sticky events can be pending continuously. - * And streamiddemux only receive gap events continuously. - * Thus, pipeline can not be constructed completely. - * For support it, need to handle GAP events likes buffers. - */ -GST_START_TEST (test_funnel_gap_event) -{ - struct TestData td; - guint probe = 0; - - setup_test_objects (&td, chain_ok); - - nb_stream_start_event = 0; - nb_caps_event = 0; - nb_segment_event = 0; - nb_gap_event = 0; - bufcount = 0; - - probe = gst_pad_add_probe (td.mysink, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, - (GstPadProbeCallback) event_counter, NULL, NULL); - - /* push a gap event to srcpad1 to push sticky events */ - fail_unless (gst_pad_push_event (td.mysrc1, gst_event_new_gap (0, - GST_SECOND))); - - fail_unless (nb_stream_start_event == 1); - fail_unless (nb_caps_event == 1); - fail_unless (nb_segment_event == 1); - fail_unless (nb_gap_event == 1); - - /* push a gap event to srcpad2 to push sticky events */ - fail_unless (gst_pad_push_event (td.mysrc2, gst_event_new_gap (0, - GST_SECOND))); - - fail_unless (nb_stream_start_event == 2); - fail_unless (nb_caps_event == 2); - fail_unless (nb_segment_event == 2); - fail_unless (nb_gap_event == 2); - - /* push a gap event to srcpad2 */ - fail_unless (gst_pad_push_event (td.mysrc2, gst_event_new_gap (0, - GST_SECOND))); - - fail_unless (nb_stream_start_event == 2); - fail_unless (nb_caps_event == 2); - fail_unless (nb_segment_event == 2); - fail_unless (nb_gap_event == 3); - - /* push a gap event to srcpad1 */ - fail_unless (gst_pad_push_event (td.mysrc1, gst_event_new_gap (0, - GST_SECOND))); - - fail_unless (nb_stream_start_event == 3); - fail_unless (nb_caps_event == 3); - fail_unless (nb_segment_event == 3); - fail_unless (nb_gap_event == 4); - - /* push buffer */ - fail_unless (gst_pad_push (td.mysrc1, gst_buffer_new ()) == GST_FLOW_OK); - fail_unless (gst_pad_push (td.mysrc2, gst_buffer_new ()) == GST_FLOW_OK); - - fail_unless (nb_stream_start_event == 4); - fail_unless (nb_caps_event == 4); - fail_unless (nb_segment_event == 4); - fail_unless (nb_gap_event == 4); - fail_unless (bufcount == 2); - - gst_pad_remove_probe (td.mysink, probe); - - release_test_objects (&td); -} - -GST_END_TEST; - GST_START_TEST (test_funnel_stress) { GstHarness *h0 = gst_harness_new_with_padnames ("funnel", "sink_0", "src"); @@ -407,6 +279,194 @@ GST_START_TEST (test_funnel_stress) GST_END_TEST; +GST_START_TEST (test_funnel_event_handling) +{ + GstHarness *h, *h0, *h1; + GstEvent *event; + GstCaps *mycaps0, *mycaps1, *caps; + + h = gst_harness_new_with_padnames ("funnel", NULL, "src"); + + /* request a sinkpad, with some caps */ + h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); + mycaps0 = gst_caps_new_empty_simple ("mycaps0"); + gst_harness_set_src_caps (h0, gst_caps_ref (mycaps0)); + + /* request a second sinkpad, also with caps */ + h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); + mycaps1 = gst_caps_new_empty_simple ("mycaps1"); + gst_harness_set_src_caps (h1, gst_caps_ref (mycaps1)); + + /* push a buffer on the first pad */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h0, gst_buffer_new ())); + gst_buffer_unref (gst_harness_pull (h)); + + /* verify stream-start */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_STREAM_START); + gst_event_unref (event); + + /* verify caps "mycaps0" */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_CAPS); + gst_event_parse_caps (event, &caps); + gst_check_caps_equal (mycaps0, caps); + gst_caps_unref (caps); + gst_event_unref (event); + + /* verify segment */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_SEGMENT); + gst_event_unref (event); + + /* verify a custom event pushed on second pad does not make it through, + since active pad is the first pad */ + fail_unless (gst_harness_push_event (h1, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new_empty ("test")))); + fail_if (gst_harness_try_pull_event (h)); + + /* but same event on the first pad will make it through */ + fail_unless (gst_harness_push_event (h0, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new_empty ("test")))); + gst_event_unref (gst_harness_pull_event (h)); + + /* push a buffer on the second pad */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h1, gst_buffer_new ())); + gst_buffer_unref (gst_harness_pull (h)); + + /* verify stream-start */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_STREAM_START); + gst_event_unref (event); + + /* verify caps "mycaps1" */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_CAPS); + gst_event_parse_caps (event, &caps); + gst_check_caps_equal (mycaps1, caps); + gst_caps_unref (caps); + gst_event_unref (event); + + /* verify segment */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_SEGMENT); + gst_event_unref (event); + + /* verify a custom event pushed on first pad does not make it through, + since active pad is the second pad */ + fail_unless (gst_harness_push_event (h0, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new_empty ("test")))); + fail_if (gst_harness_try_pull_event (h)); + + /* but same event on the second pad will make it through */ + fail_unless (gst_harness_push_event (h1, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new_empty ("test")))); + gst_event_unref (gst_harness_pull_event (h)); + + gst_harness_teardown (h); + gst_harness_teardown (h0); + gst_harness_teardown (h1); +} + +GST_END_TEST; + +GST_START_TEST (test_funnel_custom_sticky) +{ + GstHarness *h, *h0, *h1; + GstEvent *event; + const GstStructure *s; + const gchar *value = NULL; + + h = gst_harness_new_with_padnames ("funnel", NULL, "src"); + + /* request a sinkpad, with some caps */ + h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); + gst_harness_set_src_caps_str (h0, "mycaps0"); + + /* request a second sinkpad, also with caps */ + h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); + gst_harness_set_src_caps_str (h1, "mycaps1"); + + while ((event = gst_harness_try_pull_event (h))) + gst_event_unref (event); + + fail_unless (gst_harness_push_event (h0, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM_STICKY, + gst_structure_new ("test", "key", G_TYPE_STRING, "value0", + NULL)))); + + fail_unless (gst_harness_push_event (h1, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM_STICKY, + gst_structure_new ("test", "key", G_TYPE_STRING, "value1", + NULL)))); + + /* Send a buffer through first pad, expect the event to be the first one */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h0, gst_buffer_new ())); + for (;;) { + event = gst_harness_pull_event (h); + if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) + break; + gst_event_unref (event); + } + s = gst_event_get_structure (event); + fail_unless (s); + fail_unless (gst_structure_has_name (s, "test")); + value = gst_structure_get_string (s, "key"); + fail_unless_equals_string (value, "value0"); + gst_event_unref (event); + gst_buffer_unref (gst_harness_pull (h)); + + /* Send a buffer through second pad, expect the event to be the second one + */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h1, gst_buffer_new ())); + for (;;) { + event = gst_harness_pull_event (h); + if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) + break; + gst_event_unref (event); + } + s = gst_event_get_structure (event); + fail_unless (s); + fail_unless (gst_structure_has_name (s, "test")); + value = gst_structure_get_string (s, "key"); + fail_unless_equals_string (value, "value1"); + gst_event_unref (event); + gst_buffer_unref (gst_harness_pull (h)); + + /* Send a buffer through first pad, expect the event to again be the first + * one + */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h0, gst_buffer_new ())); + for (;;) { + event = gst_harness_pull_event (h); + if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) + break; + gst_event_unref (event); + } + s = gst_event_get_structure (event); + fail_unless (s); + fail_unless (gst_structure_has_name (s, "test")); + value = gst_structure_get_string (s, "key"); + fail_unless_equals_string (value, "value0"); + gst_event_unref (event); + gst_buffer_unref (gst_harness_pull (h)); + + gst_harness_teardown (h); + gst_harness_teardown (h0); + gst_harness_teardown (h1); +} + +GST_END_TEST; + static Suite * funnel_suite (void) @@ -417,8 +477,9 @@ funnel_suite (void) tc_chain = tcase_create ("funnel simple"); tcase_add_test (tc_chain, test_funnel_simple); tcase_add_test (tc_chain, test_funnel_eos); - tcase_add_test (tc_chain, test_funnel_gap_event); tcase_add_test (tc_chain, test_funnel_stress); + tcase_add_test (tc_chain, test_funnel_event_handling); + tcase_add_test (tc_chain, test_funnel_custom_sticky); suite_add_tcase (s, tc_chain); return s; From d0944d297be92f8387c57dc5b17d64256239fce4 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 23 Feb 2023 11:41:02 +0100 Subject: [PATCH 279/339] gstaudiodecoder: implemented DTX API Subclasses can now start DTX by calling gst_audio_decoder_start_dtx(), this will spawn a task on the decoders srcpad and generate silent buffers until a new audio packet or a gap event arrives. An API for checking if the decoder is running DTX has been also added in case subclasses need context. --- .../gst-libs/gst/audio/gstaudiodecoder.c | 228 +++++++++++++++- .../gst-libs/gst/audio/gstaudiodecoder.h | 9 + .../tests/check/libs/audiodecoder.c | 246 ++++++++++++++++-- 3 files changed, 463 insertions(+), 20 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c index ec61c85cc53..b97bd2809d4 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c @@ -264,6 +264,9 @@ struct _GstAudioDecoderPrivate /* context storage */ GstAudioDecoderContext ctx; + /* the cached upstream latency */ + GstClockTime us_latency; + /* properties */ GstClockTime latency; GstClockTime reported_min_latency; @@ -272,6 +275,13 @@ struct _GstAudioDecoderPrivate gboolean drainable; gboolean needs_format; + /* dtx */ + gboolean dtx_running; + GMutex dtx_lock; + GstClockID dtx_clock_id; + GstClockTime last_pts; + GstClockTime last_buffer_duration; + /* pending serialized sink events, will be sent from finish_frame() */ GList *pending_events; @@ -279,6 +289,10 @@ struct _GstAudioDecoderPrivate gboolean use_default_pad_acceptcaps; }; +#define GST_AUDIO_DECODER_DTX_GET_LOCK(dec) (&dec->priv->dtx_lock) +#define GST_AUDIO_DECODER_DTX_LOCK(dec) (g_mutex_lock (GST_AUDIO_DECODER_DTX_GET_LOCK (dec))) +#define GST_AUDIO_DECODER_DTX_UNLOCK(dec) (g_mutex_unlock (GST_AUDIO_DECODER_DTX_GET_LOCK (dec))) + /* cached quark to avoid contention on the global quark table lock */ #define META_TAG_AUDIO meta_tag_audio_quark static GQuark meta_tag_audio_quark; @@ -309,8 +323,11 @@ static GstFlowReturn gst_audio_decoder_chain (GstPad * pad, GstObject * parent, GstBuffer * buf); static gboolean gst_audio_decoder_src_query (GstPad * pad, GstObject * parent, GstQuery * query); +static gboolean gst_audio_decoder_src_activate_mode (G_GNUC_UNUSED GstPad * pad, + GstObject * parent, GstPadMode mode, gboolean active); static gboolean gst_audio_decoder_sink_query (GstPad * pad, GstObject * parent, GstQuery * query); +static void gst_audio_decoder_stop_dtx (GstAudioDecoder * dec); static void gst_audio_decoder_reset (GstAudioDecoder * dec, gboolean full); static gboolean gst_audio_decoder_decide_allocation_default (GstAudioDecoder * @@ -495,6 +512,8 @@ gst_audio_decoder_init (GstAudioDecoder * dec, GstAudioDecoderClass * klass) GST_DEBUG_FUNCPTR (gst_audio_decoder_src_event)); gst_pad_set_query_function (dec->srcpad, GST_DEBUG_FUNCPTR (gst_audio_decoder_src_query)); + gst_pad_set_activatemode_function (dec->srcpad, + GST_DEBUG_FUNCPTR (gst_audio_decoder_src_activate_mode)); gst_element_add_pad (GST_ELEMENT (dec), dec->srcpad); GST_DEBUG_OBJECT (dec, "srcpad created"); @@ -503,6 +522,7 @@ gst_audio_decoder_init (GstAudioDecoder * dec, GstAudioDecoderClass * klass) g_queue_init (&dec->priv->frames); g_rec_mutex_init (&dec->stream_lock); + g_mutex_init (GST_AUDIO_DECODER_DTX_GET_LOCK (dec)); /* property default */ dec->priv->latency = DEFAULT_LATENCY; @@ -607,6 +627,7 @@ gst_audio_decoder_finalize (GObject * object) } g_rec_mutex_clear (&dec->stream_lock); + g_mutex_clear (GST_AUDIO_DECODER_DTX_GET_LOCK (dec)); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -1434,7 +1455,7 @@ gst_audio_decoder_finish_frame_or_subframe (GstAudioDecoder * dec, if (G_LIKELY (!priv->force)) { GST_DEBUG_OBJECT (dec, "received more decoded frames %d than provided %d", frames, - priv->frames.length); + priv->frames.length); } frames = priv->frames.length; } @@ -1531,6 +1552,9 @@ gst_audio_decoder_finish_frame_or_subframe (GstAudioDecoder * dec, GST_FRAMES_TO_CLOCK_TIME (samples, ctx->info.rate); } + priv->last_pts = GST_BUFFER_PTS (buf); + priv->last_buffer_duration = GST_BUFFER_DURATION (buf); + if (klass->transform_meta) { if (inbufs.length) { GList *l; @@ -1657,6 +1681,124 @@ gst_audio_decoder_handle_frame (GstAudioDecoder * dec, return klass->handle_frame (dec, buffer); } +/* called with GST_AUDIO_DECODER_DTX_LOCK held */ +static void +gst_audio_decoder_dtx_wait_unlocked (GstAudioDecoder * dec, GstClockTime time) +{ + GstClock *clock = GST_ELEMENT_CLOCK (dec); + + if (!clock) + return; + + dec->priv->dtx_clock_id = gst_clock_new_single_shot_id (clock, time); + GST_LOG_OBJECT (dec, "Waiting for %" GST_TIME_FORMAT, GST_TIME_ARGS (time)); + + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + gst_clock_id_wait (dec->priv->dtx_clock_id, NULL); + GST_AUDIO_DECODER_DTX_LOCK (dec); + + gst_clock_id_unref (dec->priv->dtx_clock_id); + dec->priv->dtx_clock_id = NULL; +} + +static void +gst_audio_decoder_dtx_task_func (gpointer user_data) +{ + GstAudioDecoder *dec = GST_AUDIO_DECODER_CAST (user_data); + GstAudioDecoderClass *klass = GST_AUDIO_DECODER_GET_CLASS (dec); + GstClockTime pts; + GstClockTime base_time; + + base_time = gst_element_get_base_time (GST_ELEMENT (dec)); + pts = dec->priv->last_pts; + + if (!GST_CLOCK_TIME_IS_VALID (pts)) { + GST_WARNING_OBJECT (dec, "Last pts is invalid, can't start DTX"); + gst_pad_pause_task (dec->srcpad); + return; + } + + GST_AUDIO_DECODER_DTX_LOCK (dec); + while (dec->priv->dtx_running) { + GstFlowReturn ret; + GstBuffer *buf; + + pts += dec->priv->us_latency; + pts += dec->priv->last_buffer_duration; + + + gst_audio_decoder_dtx_wait_unlocked (dec, base_time + pts); + + if (!dec->priv->dtx_running) + break; + + GST_DEBUG_OBJECT (dec, + "producing DTX packet with timestamp %" GST_TIME_FORMAT, + GST_TIME_ARGS (pts)); + + /* same is in handle_gap() we tell the subclass to conceal */ + buf = gst_buffer_new (); + GST_BUFFER_PTS (buf) = pts; + GST_BUFFER_DURATION (buf) = dec->priv->last_buffer_duration; + + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + GST_AUDIO_DECODER_STREAM_LOCK (dec); + + ret = gst_audio_decoder_handle_frame (dec, klass, buf); + + GST_AUDIO_DECODER_STREAM_UNLOCK (dec); + GST_AUDIO_DECODER_DTX_LOCK (dec); + + if (ret != GST_FLOW_OK) + break; + } + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + + gst_pad_pause_task (dec->srcpad); +} + +/* called with GST_AUDIO_DECODER_DTX_LOCK held */ +static void +gst_audio_decoder_start_dtx_unlocked (GstAudioDecoder * dec) +{ + if (dec->priv->dtx_running) { + GST_ERROR_OBJECT (dec, "Can't double start the DTX task!"); + g_assert_not_reached (); + return; + } + + GST_INFO_OBJECT (dec, "Starting DTX"); + dec->priv->dtx_running = TRUE; + + if (!gst_pad_start_task (dec->srcpad, gst_audio_decoder_dtx_task_func, dec, + NULL)) + g_assert_not_reached (); +} + +static void +gst_audio_decoder_stop_dtx (GstAudioDecoder * dec) +{ + GST_AUDIO_DECODER_DTX_LOCK (dec); + + if (!dec->priv->dtx_running) { + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + return; + } + + GST_INFO_OBJECT (dec, "Stopping DTX"); + dec->priv->dtx_running = FALSE; + + if (dec->priv->dtx_clock_id) { + gst_clock_id_unschedule (dec->priv->dtx_clock_id); + g_assert (dec->priv->dtx_clock_id); + } + + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + + if (!gst_pad_stop_task (dec->srcpad)) + g_assert_not_reached (); +} + /* maybe subclass configurable instead, but this allows for a whole lot of * raw samples, so at least quite some encoded ... */ #define GST_AUDIO_DECODER_MAX_SYNC 10 * 8 * 2 * 1024 @@ -2094,6 +2236,8 @@ gst_audio_decoder_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GST_TIME_ARGS (GST_BUFFER_PTS (buffer)), GST_TIME_ARGS (GST_BUFFER_DURATION (buffer))); + gst_audio_decoder_stop_dtx (dec); + GST_AUDIO_DECODER_STREAM_LOCK (dec); if (G_UNLIKELY (dec->priv->ctx.input_caps == NULL && dec->priv->needs_format)) @@ -2274,6 +2418,8 @@ gst_audio_decoder_handle_gap (GstAudioDecoder * dec, GstEvent * event) GstClockTime timestamp, duration; gboolean needs_reconfigure = FALSE; + gst_audio_decoder_stop_dtx (dec); + /* Ensure we have caps first */ GST_AUDIO_DECODER_STREAM_LOCK (dec); if (!GST_AUDIO_INFO_IS_VALID (&dec->priv->ctx.info)) { @@ -2294,7 +2440,6 @@ gst_audio_decoder_handle_gap (GstAudioDecoder * dec, GstEvent * event) gst_pad_mark_reconfigure (dec->srcpad); } } - GST_AUDIO_DECODER_STREAM_UNLOCK (dec); gst_event_parse_gap (event, ×tamp, &duration); @@ -2337,6 +2482,9 @@ gst_audio_decoder_handle_gap (GstAudioDecoder * dec, GstEvent * event) gst_event_unref (event); } } + + GST_AUDIO_DECODER_STREAM_UNLOCK (dec); + return ret; } @@ -3063,6 +3211,9 @@ gst_audio_decoder_src_query_default (GstAudioDecoder * dec, GstQuery * query) max_latency = -1; else max_latency += dec->priv->ctx.max_latency; + + /* store the upstream min latency */ + dec->priv->us_latency = min_latency; GST_OBJECT_UNLOCK (dec); gst_query_set_latency (query, live, min_latency, max_latency); @@ -3077,6 +3228,21 @@ gst_audio_decoder_src_query_default (GstAudioDecoder * dec, GstQuery * query) return res; } +static gboolean +gst_audio_decoder_src_activate_mode (G_GNUC_UNUSED GstPad * pad, + GstObject * parent, GstPadMode mode, gboolean active) +{ + GstAudioDecoder *dec; + + dec = GST_AUDIO_DECODER (parent); + GST_LOG_OBJECT (dec, "activate mode %d active %d", mode, active); + + if (active == FALSE) + gst_audio_decoder_stop_dtx (dec); + + return TRUE; +} + static gboolean gst_audio_decoder_src_query (GstPad * pad, GstObject * parent, GstQuery * query) { @@ -3105,6 +3271,8 @@ gst_audio_decoder_stop (GstAudioDecoder * dec) klass = GST_AUDIO_DECODER_GET_CLASS (dec); + gst_audio_decoder_stop_dtx (dec); + if (klass->stop) { ret = klass->stop (dec); } @@ -3616,6 +3784,29 @@ gst_audio_decoder_get_min_latency (GstAudioDecoder * dec) return result; } +/** + * gst_audio_decoder_get_upstream_latency: + * @dec: a #GstAudioDecoder + * + * Returns: the upstream latency. + * + * MT safe. + */ +GstClockTime +gst_audio_decoder_get_upstream_latency (GstAudioDecoder * dec) +{ + GstClockTime result; + + g_return_val_if_fail (GST_IS_AUDIO_DECODER (dec), FALSE); + + GST_OBJECT_LOCK (dec); + result = dec->priv->us_latency; + GST_OBJECT_UNLOCK (dec); + + return result; +} + + /** * gst_audio_decoder_set_tolerance: * @dec: a #GstAudioDecoder @@ -3754,6 +3945,39 @@ gst_audio_decoder_get_needs_format (GstAudioDecoder * dec) return result; } +/** + * gst_audio_decoder_start_dtx: + * @dec: a #GstAudioDecoder + * + * Starts the DTX task. The audio decoder will spawn a task that will generate + * the silent buffers to cover the period of time we haven't received frames for. + * + * MT safe. + */ +void +gst_audio_decoder_start_dtx (GstAudioDecoder * dec) +{ + g_return_if_fail (GST_IS_AUDIO_DECODER (dec)); + + GST_AUDIO_DECODER_DTX_LOCK (dec); + gst_audio_decoder_start_dtx_unlocked (dec); + GST_AUDIO_DECODER_DTX_UNLOCK (dec); +} + +/** + * gst_audio_decoder_dtx_running: + * @dec: a #GstAudioDecoder + * + * Returns: TRUE if DTX has started on this decoder. + * + * MT safe. + */ +gboolean +gst_audio_decoder_dtx_running (GstAudioDecoder * dec) +{ + return dec->priv->dtx_running; +} + /** * gst_audio_decoder_merge_tags: * @dec: a #GstAudioDecoder diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h index 03fb675390e..0653e45f231 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h @@ -413,6 +413,9 @@ void gst_audio_decoder_set_min_latency (GstAudioDecoder * dec, GST_AUDIO_API GstClockTime gst_audio_decoder_get_min_latency (GstAudioDecoder * dec); +GST_AUDIO_API +GstClockTime gst_audio_decoder_get_upstream_latency (GstAudioDecoder * dec); + GST_AUDIO_API void gst_audio_decoder_set_tolerance (GstAudioDecoder * dec, GstClockTime tolerance); @@ -434,6 +437,12 @@ void gst_audio_decoder_set_needs_format (GstAudioDecoder * dec, GST_AUDIO_API gboolean gst_audio_decoder_get_needs_format (GstAudioDecoder * dec); +GST_AUDIO_API +void gst_audio_decoder_start_dtx (GstAudioDecoder * dec); + +GST_AUDIO_API +gboolean gst_audio_decoder_dtx_running (GstAudioDecoder * dec); + GST_AUDIO_API void gst_audio_decoder_get_allocator (GstAudioDecoder * dec, GstAllocator ** allocator, diff --git a/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c b/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c index 8f69f53208d..c8fb165370c 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c +++ b/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c @@ -86,7 +86,7 @@ G_DEFINE_TYPE (GstAudioDecoderTester, gst_audio_decoder_tester, GST_TYPE_AUDIO_DECODER); static gboolean -gst_audio_decoder_tester_start (GstAudioDecoder * dec) +gst_audio_decoder_tester_start (G_GNUC_UNUSED GstAudioDecoder * dec) { return TRUE; } @@ -103,7 +103,8 @@ gst_audio_decoder_tester_stop (GstAudioDecoder * dec) } static void -gst_audio_decoder_tester_flush (GstAudioDecoder * dec, gboolean hard) +gst_audio_decoder_tester_flush (G_GNUC_UNUSED GstAudioDecoder * dec, + G_GNUC_UNUSED gboolean hard) { } @@ -137,11 +138,16 @@ gst_audio_decoder_tester_handle_frame (GstAudioDecoder * dec, GstFlowReturn ret = GST_FLOW_OK; gboolean do_plc = gst_audio_decoder_get_plc (dec) && gst_audio_decoder_get_plc_aware (dec); + gboolean is_dtx_frame; if (buffer == NULL || (!do_plc && gst_buffer_get_size (buffer) == 0)) return GST_FLOW_OK; gst_buffer_ref (buffer); + + /* for testing purposes, 1byte size buffers signals the start of DTX */ + is_dtx_frame = (gst_buffer_get_size (buffer) == 1); + if (tester->setoutputformat_on_decoding) { GstCaps *caps; GstAudioInfo info; @@ -157,7 +163,7 @@ gst_audio_decoder_tester_handle_frame (GstAudioDecoder * dec, if ((tester->delay_decoding && tester->prev_buf != NULL) || !tester->delay_decoding) { gsize buf_num = tester->delay_decoding ? 2 : 1; - gint i; + gsize i; for (i = 0; i != buf_num; ++i) { GstBuffer *cur_buf = buf_num == 1 || i != 0 ? buffer : tester->prev_buf; @@ -168,7 +174,7 @@ gst_audio_decoder_tester_handle_frame (GstAudioDecoder * dec, g_assert (size == sizeof (guint64)); data = g_malloc0 (size); - if (map.size) { + if (map.size && !is_dtx_frame) { g_assert_cmpint (map.size, >=, sizeof (guint64)); memcpy (data, map.data, sizeof (guint64)); } @@ -188,6 +194,9 @@ gst_audio_decoder_tester_handle_frame (GstAudioDecoder * dec, tester->delay_decoding = FALSE; } + if (is_dtx_frame) + gst_audio_decoder_start_dtx (dec); + if (tester->prev_buf) gst_buffer_unref (tester->prev_buf); tester->prev_buf = NULL; @@ -226,7 +235,7 @@ gst_audio_decoder_tester_class_init (GstAudioDecoderTesterClass * klass) } static void -gst_audio_decoder_tester_init (GstAudioDecoderTester * tester) +gst_audio_decoder_tester_init (G_GNUC_UNUSED GstAudioDecoderTester * tester) { } @@ -271,6 +280,50 @@ create_test_buffer (guint64 num) return buffer; } +static float +get_avg_buf_power (float *data, gint samples) +{ + float sum_power = 0.0f; + for (int s = 0; s < samples; s++) + sum_power += data[s] * data[s]; + + return sum_power / (float) samples; +} + +static void +fail_unless_silent_buffer (GstBuffer * buf) +{ + GstMapInfo map; + fail_unless (gst_buffer_map (buf, &map, GST_MAP_READ)); + float power = get_avg_buf_power ((float *) map.data, + (guint) map.size / sizeof (float)); + fail_unless_equals_float (0.0f, power); + gst_buffer_unmap (buf, &map); + gst_buffer_unref (buf); +} + +static void +fail_unless_silent_buffers (GstHarness * h, gint count) +{ + for (gint i = 0; i < count; i++) { + GstBuffer *buf; + + gst_harness_crank_single_clock_wait (h); + buf = gst_harness_pull (h); + fail_unless_silent_buffer (buf); + } +} + +static GstBuffer * +create_dtx_buffer (guint64 num) +{ + /* 1byte buffer signals DTX */ + GstBuffer *buf = create_test_buffer (num); + gst_buffer_set_size (buf, 1); + gst_buffer_memset (buf, 0, 0, 1); + return buf; +} + #define NUM_BUFFERS 10 GST_START_TEST (audiodecoder_playback) @@ -804,7 +857,8 @@ GST_END_TEST; #define GETCAPS_CAPS_STR "audio/x-test-custom, somefield=(string)getcaps" static GstCaps * -_custom_audio_decoder_getcaps (GstAudioDecoder * dec, GstCaps * filter) +_custom_audio_decoder_getcaps (G_GNUC_UNUSED GstAudioDecoder * dec, + G_GNUC_UNUSED GstCaps * filter) { return gst_caps_from_string (GETCAPS_CAPS_STR); } @@ -1063,16 +1117,16 @@ GST_START_TEST (audiodecoder_plc_on_gap_event) pts = gst_util_uint64_scale_round (0, GST_SECOND, TEST_MSECS_PER_SAMPLE); gst_harness_push (h, create_test_buffer (0)); buf = gst_harness_pull (h); - fail_unless_equals_int (pts, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); pts = gst_util_uint64_scale_round (1, GST_SECOND, TEST_MSECS_PER_SAMPLE); gst_harness_push_event (h, gst_event_new_gap (pts, dur)); buf = gst_harness_pull (h); - fail_unless_equals_int (pts, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (!GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); @@ -1081,8 +1135,8 @@ GST_START_TEST (audiodecoder_plc_on_gap_event) GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DISCONT); gst_harness_push (h, buf); buf = gst_harness_pull (h); - fail_unless_equals_int (pts, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (!GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); gst_harness_teardown (h); @@ -1106,8 +1160,8 @@ GST_START_TEST (audiodecoder_plc_on_gap_event_with_delay) pts0 = gst_util_uint64_scale_round (0, GST_SECOND, TEST_MSECS_PER_SAMPLE);; gst_harness_push (h, create_test_buffer (0)); buf = gst_harness_pull (h); - fail_unless_equals_int (pts0, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); @@ -1121,14 +1175,14 @@ GST_START_TEST (audiodecoder_plc_on_gap_event_with_delay) GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DISCONT); gst_harness_push (h, buf); buf = gst_harness_pull (h); - fail_unless_equals_int (pts0, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (!GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); buf = gst_harness_pull (h); - fail_unless_equals_int (pts1, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts1, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (!GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); gst_harness_teardown (h); @@ -1136,6 +1190,156 @@ GST_START_TEST (audiodecoder_plc_on_gap_event_with_delay) GST_END_TEST; +GST_START_TEST (audiodecoder_dtx) +{ + guint pkt = 0; + GstClockTime pts, dur; + GstHarness *h = setup_audiodecodertester (NULL, NULL); + gst_audio_decoder_set_plc_aware (GST_AUDIO_DECODER (h->element), TRUE); + gst_audio_decoder_set_plc (GST_AUDIO_DECODER (h->element), TRUE); + + /* start/stop DTX 3 times */ + for (gint i = 0; i < 3; i++) { + GstBuffer *buf; + + buf = create_dtx_buffer (pkt++); + pts = GST_BUFFER_PTS (buf); + dur = GST_BUFFER_DURATION (buf); + fail_unless_equals_int64 (GST_FLOW_OK, gst_harness_push (h, buf)); + + /* the DTX packet goes through */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); + fail_unless_silent_buffer (buf); + + /* expect some silent buffers from the DTX thread */ + fail_unless_silent_buffers (h, 2); + /* advance two packets */ + pkt += 2; + + /* and audio buffer arrives, the decoder should stop DTX */ + buf = create_test_buffer (pkt++); + pts = GST_BUFFER_PTS (buf); + dur = GST_BUFFER_DURATION (buf); + fail_unless_equals_int64 (GST_FLOW_OK, gst_harness_push (h, buf)); + + /* that audio buffer goes through */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* the decoder does not produce more packets */ + fail_unless (gst_harness_try_pull (h) == NULL); + } + + gst_harness_teardown (h); +} + +GST_END_TEST; + +static const int audiodecoder_lost_pkt_stops_dtx_latencies[] = { + 0, 10, 20, 50, 100, 300, 500, 600, +}; + +GST_START_TEST (audiodecoder_lost_pkt_stops_dtx) +{ + gint latency_ms = audiodecoder_lost_pkt_stops_dtx_latencies[__i__]; + GstClockTime upstream_latency = latency_ms * GST_MSECOND; + GstClockTime pts, dur; + GstClockTime lost_pts, lost_dur; + GstBuffer *buf; + GstHarness *h = setup_audiodecodertester (NULL, NULL); + + gst_audio_decoder_set_plc_aware (GST_AUDIO_DECODER (h->element), TRUE); + gst_audio_decoder_set_plc (GST_AUDIO_DECODER (h->element), TRUE); + gst_harness_set_upstream_latency (h, upstream_latency); + + /* start with DTX in */ + buf = create_dtx_buffer (0); + pts = GST_BUFFER_PTS (buf); + dur = GST_BUFFER_DURATION (buf); + fail_unless_equals_int64 (GST_FLOW_OK, gst_harness_push (h, buf)); + + /* the DTX packet goes through */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); + fail_unless_silent_buffer (buf); + + /* lets receive 3 silent buffers */ + fail_unless_silent_buffers (h, 3); + + /* buffer 4 is lost, dtx should be stopped now */ + buf = create_test_buffer (4); + lost_pts = GST_BUFFER_PTS (buf); + lost_dur = GST_BUFFER_DURATION (buf); + gst_harness_push_event (h, gst_event_new_gap (lost_pts, lost_dur)); + gst_buffer_unref (buf); + + /* audio buffer 5 arrives */ + buf = create_test_buffer (5); + pts = GST_BUFFER_PTS (buf); + dur = GST_BUFFER_DURATION (buf); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + + /* expect a conceal buffer 4 is generated for the lost one */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (lost_pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (lost_dur, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* and expect buffer 5 */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} + +GST_END_TEST; + +static GstBuffer * +audiodecoder_dtx_stress_prepare_buffer (G_GNUC_UNUSED GstHarness * h, + gpointer data) +{ + gint *i = data; + GstBuffer *buf = create_test_buffer (*i); + *i = *i + 1; + return buf; +} + +GST_START_TEST (audiodecoder_dtx_stress_push) +{ + gint i = 0; + GstCaps *caps; + GstSegment segment; + GstHarnessThread *push_t; + GstHarness *h = setup_audiodecodertester (NULL, NULL); + + gst_audio_decoder_set_plc_aware (GST_AUDIO_DECODER (h->element), TRUE); + gst_audio_decoder_set_plc (GST_AUDIO_DECODER (h->element), TRUE); + gst_harness_set_drop_buffers (h, TRUE); + gst_harness_use_systemclock (h); + + caps = gst_caps_new_simple ("audio/x-test-custom", + "channels", G_TYPE_INT, 2, "rate", G_TYPE_INT, 44100, NULL); + gst_segment_init (&segment, GST_FORMAT_TIME); + + push_t = gst_harness_stress_push_buffer_with_cb_start (h, caps, &segment, + audiodecoder_dtx_stress_prepare_buffer, &i, NULL); + + g_usleep (1 * G_USEC_PER_SEC); + + gst_harness_stress_thread_stop (push_t); + gst_caps_unref (caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + static Suite * gst_audiodecoder_suite (void) { @@ -1166,6 +1370,12 @@ gst_audiodecoder_suite (void) tcase_add_test (tc, audiodecoder_plc_on_gap_event); tcase_add_test (tc, audiodecoder_plc_on_gap_event_with_delay); + suite_add_tcase (s, tc = tcase_create ("dtx")); + tcase_add_test (tc, audiodecoder_dtx); + tcase_add_loop_test (tc, audiodecoder_lost_pkt_stops_dtx, 0, + G_N_ELEMENTS (audiodecoder_lost_pkt_stops_dtx_latencies)); + tcase_add_test (tc, audiodecoder_dtx_stress_push); + return s; } From dde703b652fb576eaaae1ed5043bbbf8958ef44e Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 27 Jan 2023 16:11:16 +0100 Subject: [PATCH 280/339] SCTP: fixed scanbuild logic issues in user_mbuf.c --- .../gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c index 9e566e30259..948ce7e06c8 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c @@ -122,6 +122,7 @@ m_get(int how, short type) /* Mbuf master zone, zone_mbuf, has already been * created in mbuf_initialize() */ mret = SCTP_ZONE_GET(zone_mbuf, struct mbuf); + memset(mret, 0, sizeof (struct mbuf)); #if defined(SCTP_SIMPLE_ALLOCATOR) mb_ctor_mbuf(mret, &mbuf_mb_args, 0); #endif @@ -162,6 +163,7 @@ m_gethdr(int how, short type) mbuf_mb_args.type = type; #endif mret = SCTP_ZONE_GET(zone_mbuf, struct mbuf); + memset(mret, 0, sizeof (struct mbuf)); #if defined(SCTP_SIMPLE_ALLOCATOR) mb_ctor_mbuf(mret, &mbuf_mb_args, 0); #endif From 5a82caf9b37024e656dfbfd861c12ace757c9a91 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 3 Feb 2023 15:53:05 +0100 Subject: [PATCH 281/339] SCTP: refactor to fix encoder use-after-free bug Because of the nature of not having a clear ownership between the association and the elements (encoder or decoder) is not possible to rely on the associations signals to have a valid reference on the encoder. This also applies for the decoder. Since g_signal_connect_object is not thread-safe (more on this in the documentation) and we can't hold the association mutex while emitting a signal, it is completely possible -as our stress tests demonstrate- that we ran into data races. To fix this situation, I introduce proper callback structures per element context and inside the association we always keep a reference to the element while calling the related callback. --- .../gst-plugins-bad/ext/sctp/gstsctpdec.c | 46 ++--- .../gst-plugins-bad/ext/sctp/gstsctpdec.h | 2 - .../gst-plugins-bad/ext/sctp/gstsctpenc.c | 36 ++-- .../gst-plugins-bad/ext/sctp/gstsctpenc.h | 2 - .../ext/sctp/sctpassociation.c | 162 ++++++++++++------ .../ext/sctp/sctpassociation.h | 54 ++++-- 6 files changed, 176 insertions(+), 126 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index 3c29568ad17..89211d7ee02 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -163,9 +163,9 @@ static void gst_sctp_data_srcpad_loop (GstPad * pad); static gboolean configure_association (GstSctpDec * self); static void cleanup_association (GstSctpDec * self); static void on_gst_sctp_association_stream_reset (GstSctpAssociation * - gst_sctp_association, guint16 stream_id, GstSctpDec * self); + gst_sctp_association, guint16 stream_id, gpointer user_data); static void on_gst_sctp_association_restart (GstSctpAssociation * - gst_sctp_association, GstSctpDec * self); + gst_sctp_association, gpointer user_data); static void on_receive (GstSctpAssociation * gst_sctp_association, guint8 * buf, gsize length, guint16 stream_id, guint ppid, gpointer user_data); @@ -528,6 +528,7 @@ static gboolean configure_association (GstSctpDec * self) { gint state; + GstSctpAssociationDecoderCtx ctx; GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); self->sctp_association = gst_sctp_association_get (self->sctp_association_id); @@ -542,19 +543,14 @@ configure_association (GstSctpDec * self) return FALSE; } - self->signal_handler_stream_reset = - g_signal_connect_object (self->sctp_association, "stream-reset", - G_CALLBACK (on_gst_sctp_association_stream_reset), self, 0); - - self->signal_handler_association_restart = - g_signal_connect_object (self->sctp_association, "association-restart", - G_CALLBACK (on_gst_sctp_association_restart), self, 0); - g_object_set (self->sctp_association, "local-port", self->local_sctp_port, NULL); - gst_sctp_association_set_on_packet_received (self->sctp_association, - on_receive, gst_object_ref (self), gst_object_unref); + ctx.element = self; + ctx.stream_reset_cb = on_gst_sctp_association_stream_reset; + ctx.restart_cb = on_gst_sctp_association_restart; + ctx.packet_received_cb = on_receive; + gst_sctp_association_set_decoder_ctx (self->sctp_association, &ctx); GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); @@ -707,10 +703,11 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) static void on_gst_sctp_association_stream_reset (GstSctpAssociation * gst_sctp_association, - guint16 stream_id, GstSctpDec * self) + guint16 stream_id, gpointer user_data) { gchar *pad_name; GstPad *srcpad; + GstSctpDec *self = user_data; GST_DEBUG_OBJECT (self, "Stream %u reset", stream_id); @@ -733,8 +730,9 @@ on_gst_sctp_association_stream_reset (GstSctpAssociation * gst_sctp_association, static void on_gst_sctp_association_restart (GstSctpAssociation * gst_sctp_association, - GstSctpDec * self) + gpointer user_data) { + GstSctpDec *self = user_data; (void) gst_sctp_association; g_signal_emit (self, signals[SIGNAL_ASSOC_RESTART], 0); } @@ -786,26 +784,16 @@ on_receive (GstSctpAssociation * sctp_association, guint8 * buf, static void cleanup_association (GstSctpDec * self) { + GstSctpAssociationDecoderCtx ctx; + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); if (!self->sctp_association) { GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); return; } - - if (self->signal_handler_association_restart != 0) { - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_association_restart); - self->signal_handler_association_restart = 0; - } - - if (self->signal_handler_stream_reset != 0) { - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_stream_reset); - self->signal_handler_stream_reset = 0; - } - - gst_sctp_association_set_on_packet_received (self->sctp_association, NULL, - NULL, NULL); + + memset (&ctx, 0, sizeof (GstSctpAssociationDecoderCtx)); + gst_sctp_association_set_decoder_ctx (self->sctp_association, &ctx); gst_sctp_association_force_close (self->sctp_association); gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h index 82eb5d5db5a..c431e41b171 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h @@ -55,8 +55,6 @@ struct _GstSctpDec guint local_sctp_port; GstSctpAssociation *sctp_association; - gulong signal_handler_stream_reset; - gulong signal_handler_association_restart; }; struct _GstSctpDecClass diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 57faeeb5567..604b3c162cc 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -165,7 +165,7 @@ static gboolean gst_sctp_enc_src_activate_mode (GstPad * pad, GstObject * parent, GstPadMode mode, gboolean active); static void on_sctp_association_state_changed (GstSctpAssociation * - sctp_association, GParamSpec * pspec, GstSctpEnc * self); + sctp_association, GstSctpAssociationState state, gpointer user_data); static gboolean configure_association (GstSctpEnc * self); static void cleanup_association (GstSctpEnc * self); @@ -998,6 +998,7 @@ static gboolean configure_association (GstSctpEnc * self) { gint state; + GstSctpAssociationEncoderCtx ctx; GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); self->sctp_association = gst_sctp_association_get (self->sctp_association_id); @@ -1012,29 +1013,28 @@ configure_association (GstSctpEnc * self) return FALSE; } - self->signal_handler_state_changed = - g_signal_connect_object (self->sctp_association, "notify::state", - G_CALLBACK (on_sctp_association_state_changed), self, 0); - g_object_set (self->sctp_association, "remote-port", self->remote_sctp_port, "use-sock-stream", self->use_sock_stream, "aggressive-heartbeat", self->aggressive_heartbeat, NULL); - gst_sctp_association_set_on_packet_out (self->sctp_association, - on_sctp_packet_out, gst_object_ref (self), gst_object_unref); + ctx.element = self; + ctx.state_change_cb = on_sctp_association_state_changed; + ctx.packet_out_cb = on_sctp_packet_out; + gst_sctp_association_set_encoder_ctx (self->sctp_association, &ctx); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); return TRUE; } static void -on_sctp_association_state_changed (GstSctpAssociation * sctp_association, - GParamSpec * pspec, GstSctpEnc * self) +on_sctp_association_state_changed (GstSctpAssociation * + sctp_association, GstSctpAssociationState state, gpointer user_data) { - gint state; - - g_object_get (sctp_association, "state", &state, NULL); + GstSctpEnc *self = (GstSctpEnc *) user_data; + /* we demand to have a valid encoder here */ + g_assert (self); GST_DEBUG_OBJECT (self, "Association state changed to %d", state); switch (state) { @@ -1116,20 +1116,16 @@ on_sctp_packet_out (GstSctpAssociation * _association, const guint8 * buf, static void cleanup_association (GstSctpEnc * self) { + GstSctpAssociationEncoderCtx ctx; + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); if (!self->sctp_association) { GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); return; } - if (self->signal_handler_state_changed != 0) { - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_state_changed); - self->signal_handler_state_changed = 0; - } - - gst_sctp_association_set_on_packet_out (self->sctp_association, NULL, NULL, - NULL); + memset (&ctx, 0, sizeof (GstSctpAssociationEncoderCtx)); + gst_sctp_association_set_encoder_ctx (self->sctp_association, &ctx); gst_sctp_association_force_close (self->sctp_association); gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index acc1fee31a3..e8794bc25c9 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -60,8 +60,6 @@ struct _GstSctpEnc GstDataQueue *outbound_sctp_packet_queue; GQueue pending_pads; - - gulong signal_handler_state_changed; }; struct _GstSctpEncClass diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 095c6475437..69dff8dd34b 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -69,14 +69,6 @@ gst_sctp_association_state_get_type (void) G_DEFINE_TYPE (GstSctpAssociation, gst_sctp_association, G_TYPE_OBJECT); -enum -{ - SIGNAL_STREAM_RESET, - SIGNAL_ASSOC_RESTART, - LAST_SIGNAL -}; - - enum { PROP_0, @@ -91,8 +83,6 @@ enum NUM_PROPERTIES }; -static guint signals[LAST_SIGNAL] = { 0 }; - static GParamSpec *properties[NUM_PROPERTIES]; #define DEFAULT_NUMBER_OF_SCTP_STREAMS 65535 @@ -148,17 +138,6 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) gobject_class->set_property = gst_sctp_association_set_property; gobject_class->get_property = gst_sctp_association_get_property; - signals[SIGNAL_STREAM_RESET] = - g_signal_new ("stream-reset", G_OBJECT_CLASS_TYPE (klass), - G_SIGNAL_RUN_FIRST, G_STRUCT_OFFSET (GstSctpAssociationClass, - on_sctp_stream_reset), NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_UINT); - - signals[SIGNAL_ASSOC_RESTART] = - g_signal_new ("association-restart", G_OBJECT_CLASS_TYPE (klass), - G_SIGNAL_RUN_FIRST, G_STRUCT_OFFSET (GstSctpAssociationClass, - on_sctp_association_restart), NULL, NULL, g_cclosure_marshal_generic, - G_TYPE_NONE, 0); - properties[PROP_ASSOCIATION_ID] = g_param_spec_uint ("association-id", "The SCTP association-id", "The SCTP association-id.", 0, G_MAXUSHORT, DEFAULT_LOCAL_SCTP_PORT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); @@ -291,9 +270,11 @@ gst_sctp_association_set_property (GObject * object, guint prop_id, static void maybe_set_state_to_ready_unlocked (GstSctpAssociation * self) { - if ((self->state == GST_SCTP_ASSOCIATION_STATE_NEW) && - (self->local_port != 0 && self->remote_port != 0) - && (self->packet_out_cb != NULL) && (self->packet_received_cb != NULL)) { + if ((self->state == GST_SCTP_ASSOCIATION_STATE_NEW) + && (self->local_port != 0 && self->remote_port != 0) + && (self->encoder_ctx.packet_out_cb != NULL) + && (self->decoder_ctx.packet_received_cb != NULL) + && (self->encoder_ctx.state_change_cb != NULL)) { gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_READY); } @@ -477,35 +458,43 @@ gst_sctp_association_start (GstSctpAssociation * self) } void -gst_sctp_association_set_on_packet_out (GstSctpAssociation * self, - GstSctpAssociationPacketOutCb packet_out_cb, gpointer user_data, - GDestroyNotify destroy_notify) +gst_sctp_association_set_encoder_ctx (GstSctpAssociation * self, + GstSctpAssociationEncoderCtx * ctx) { g_return_if_fail (GST_SCTP_IS_ASSOCIATION (self)); g_mutex_lock (&self->association_mutex); - if (self->packet_out_destroy_notify) - self->packet_out_destroy_notify (self->packet_out_user_data); - self->packet_out_cb = packet_out_cb; - self->packet_out_user_data = user_data; - self->packet_out_destroy_notify = destroy_notify; + + if (self->encoder_ctx.element) + gst_object_unref (self->encoder_ctx.element); + + g_assert (ctx); + self->encoder_ctx = *ctx; + + if (ctx->element) + self->encoder_ctx.element = gst_object_ref (ctx->element); + maybe_set_state_to_ready_unlocked (self); g_mutex_unlock (&self->association_mutex); } void -gst_sctp_association_set_on_packet_received (GstSctpAssociation * self, - GstSctpAssociationPacketReceivedCb packet_received_cb, gpointer user_data, - GDestroyNotify destroy_notify) +gst_sctp_association_set_decoder_ctx (GstSctpAssociation * self, + GstSctpAssociationDecoderCtx * ctx) { g_return_if_fail (GST_SCTP_IS_ASSOCIATION (self)); g_mutex_lock (&self->association_mutex); - if (self->packet_received_destroy_notify) - self->packet_received_destroy_notify (self->packet_received_user_data); - self->packet_received_cb = packet_received_cb; - self->packet_received_user_data = user_data; - self->packet_received_destroy_notify = destroy_notify; + + if (self->decoder_ctx.element) + gst_object_unref (self->decoder_ctx.element); + + g_assert (ctx); + self->decoder_ctx = *ctx; + + if (ctx->element) + self->decoder_ctx.element = gst_object_ref (ctx->element); + maybe_set_state_to_ready_unlocked (self); g_mutex_unlock (&self->association_mutex); } @@ -928,8 +917,9 @@ sctp_packet_out (void *addr, void *buffer, size_t length, guint8 tos, { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (addr); - if (association_is_valid (self) && self->packet_out_cb) { - self->packet_out_cb (self, buffer, length, self->packet_out_user_data); + if (association_is_valid (self) && self->encoder_ctx.packet_out_cb) { + self->encoder_ctx.packet_out_cb (self, buffer, length, + self->encoder_ctx.element); } return 0; @@ -1092,6 +1082,51 @@ handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, g_mutex_unlock (&self->association_mutex); } +static void +gst_sctp_association_notify_restart (GstSctpAssociation * self) +{ + GstSctpAssociationRestartCb restart_cb; + gpointer user_data; + + g_mutex_lock (&self->association_mutex); + restart_cb = self->decoder_ctx.restart_cb; + user_data = self->decoder_ctx.element; + if (user_data) + gst_object_ref (user_data); + g_mutex_unlock (&self->association_mutex); + + if (restart_cb) + restart_cb (self, user_data); + + g_mutex_lock (&self->association_mutex); + if (user_data) + gst_object_unref (user_data); + g_mutex_unlock (&self->association_mutex); +} + +static void +gst_sctp_association_notify_stream_reset (GstSctpAssociation * self, + guint16 stream_id) +{ + GstSctpAssociationStreamResetCb stream_reset_cb; + gpointer user_data; + + g_mutex_lock (&self->association_mutex); + stream_reset_cb = self->decoder_ctx.stream_reset_cb; + user_data = self->decoder_ctx.element; + if (user_data) + gst_object_ref (user_data); + g_mutex_unlock (&self->association_mutex); + + if (stream_reset_cb) + stream_reset_cb (self, stream_id, user_data); + + g_mutex_lock (&self->association_mutex); + if (user_data) + gst_object_unref (user_data); + g_mutex_unlock (&self->association_mutex); +} + static void handle_association_changed (GstSctpAssociation * self, const struct sctp_assoc_change *sac) @@ -1105,7 +1140,7 @@ handle_association_changed (GstSctpAssociation * self, break; case SCTP_RESTART: GST_INFO_OBJECT (self, "SCTP event SCTP_RESTART received"); - g_signal_emit (self, signals[SIGNAL_ASSOC_RESTART], 0); + gst_sctp_association_notify_restart (self); break; case SCTP_SHUTDOWN_COMP: /* Occurs if in TCP mode when the far end sends SHUTDOWN */ @@ -1130,7 +1165,7 @@ handle_stream_reset_event (GstSctpAssociation * self, sizeof (struct sctp_stream_reset_event)) / sizeof (uint16_t); for (i = 0; i < n; i++) { if (sr->strreset_flags & SCTP_STREAM_RESET_INCOMING_SSN) { - g_signal_emit (self, signals[SIGNAL_STREAM_RESET], 0, + gst_sctp_association_notify_stream_reset (self, sr->strreset_stream_list[i]); } } @@ -1142,10 +1177,10 @@ handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid) { g_mutex_lock (&self->association_mutex); - if (self->packet_received_cb) { + if (self->decoder_ctx.packet_received_cb) { /* It's the callbacks job to free the data correctly */ - self->packet_received_cb (self, data, datalen, stream_id, ppid, - self->packet_received_user_data); + self->decoder_ctx.packet_received_cb (self, data, datalen, stream_id, ppid, + self->decoder_ctx.element); } else { /* We use this instead of a bare `free()` so that we use the `free` from * the C runtime that usrsctp was built with. This makes a difference on @@ -1161,6 +1196,8 @@ gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, GstSctpAssociationState new_state) { gboolean notify = FALSE; + GstSctpAssociationStateChangeCb callback = self->encoder_ctx.state_change_cb; + gpointer encoder = self->encoder_ctx.element; if (self->state != new_state && self->state != GST_SCTP_ASSOCIATION_STATE_ERROR) { @@ -1168,15 +1205,28 @@ gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, notify = TRUE; } - /* Unlock the mutex to emit the property change event to avoid deadlock - * if the client calls back into this object from its event handler. */ - if (notify) { - gst_sctp_association_ref (self); - g_mutex_unlock (&self->association_mutex); - g_object_notify_by_pspec (G_OBJECT (self), properties[PROP_STATE]); - g_mutex_lock (&self->association_mutex); - gst_sctp_association_unref (self); - } + /* return immediately if we don't have to notify */ + if (!notify) + return; + + /* hold a ref on the association and the encoder, so we make sure they + outlives the callback execution */ + gst_sctp_association_ref (self); + if (encoder) + gst_object_ref (encoder); + + /* release the association mutex, so other calls can be done to the + association */ + g_mutex_unlock (&self->association_mutex); + + if (callback) + callback (self, new_state, encoder); + + g_mutex_lock (&self->association_mutex); + + if (encoder) + gst_object_unref (encoder); + gst_sctp_association_unref (self); } static void diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 434c8a85b7b..3779a6d4829 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -42,6 +42,9 @@ G_BEGIN_DECLS typedef struct _GstSctpAssociation GstSctpAssociation; typedef struct _GstSctpAssociationClass GstSctpAssociationClass; +typedef struct _GstSctpAssociationEncoderCtx GstSctpAssociationEncoderCtx; +typedef struct _GstSctpAssociationDecoderCtx GstSctpAssociationDecoderCtx; + typedef enum { GST_SCTP_ASSOCIATION_STATE_NEW, @@ -61,11 +64,35 @@ typedef enum GST_SCTP_ASSOCIATION_PARTIAL_RELIABILITY_RTX = 0x0003 } GstSctpAssociationPartialReliability; +typedef void (*GstSctpAssociationPacketOutCb) (GstSctpAssociation * + sctp_association, const guint8 * data, gsize length, gpointer user_data); +typedef void (*GstSctpAssociationStateChangeCb) (GstSctpAssociation * + sctp_association, GstSctpAssociationState state, gpointer user_data); + +struct _GstSctpAssociationEncoderCtx +{ + gpointer element; + GstSctpAssociationStateChangeCb state_change_cb; + GstSctpAssociationPacketOutCb packet_out_cb; +}; + typedef void (*GstSctpAssociationPacketReceivedCb) (GstSctpAssociation * sctp_association, guint8 * data, gsize length, guint16 stream_id, guint ppid, gpointer user_data); -typedef void (*GstSctpAssociationPacketOutCb) (GstSctpAssociation * - sctp_association, const guint8 * data, gsize length, gpointer user_data); + +typedef void (*GstSctpAssociationStreamResetCb)(GstSctpAssociation * + sctp_association, guint16 stream_id, gpointer user_data); + +typedef void (*GstSctpAssociationRestartCb)(GstSctpAssociation * + sctp_association, gpointer user_data); + +struct _GstSctpAssociationDecoderCtx +{ + gpointer element; + GstSctpAssociationPacketReceivedCb packet_received_cb; + GstSctpAssociationStreamResetCb stream_reset_cb; + GstSctpAssociationRestartCb restart_cb; +}; struct _GstSctpAssociation { @@ -85,22 +112,13 @@ struct _GstSctpAssociation guint32 sctp_assoc_id; - GstSctpAssociationPacketReceivedCb packet_received_cb; - gpointer packet_received_user_data; - GDestroyNotify packet_received_destroy_notify; - - GstSctpAssociationPacketOutCb packet_out_cb; - gpointer packet_out_user_data; - GDestroyNotify packet_out_destroy_notify; + GstSctpAssociationEncoderCtx encoder_ctx; + GstSctpAssociationDecoderCtx decoder_ctx; }; struct _GstSctpAssociationClass { GObjectClass parent_class; - - void (*on_sctp_stream_reset) (GstSctpAssociation * sctp_association, - guint16 stream_id); - void (*on_sctp_association_restart) (GstSctpAssociation * sctp_association); }; GType gst_sctp_association_get_type (void); @@ -111,10 +129,12 @@ GstSctpAssociation *gst_sctp_association_ref (GstSctpAssociation * self); void gst_sctp_association_unref (GstSctpAssociation * self); gboolean gst_sctp_association_start (GstSctpAssociation * self); -void gst_sctp_association_set_on_packet_out (GstSctpAssociation * self, - GstSctpAssociationPacketOutCb packet_out_cb, gpointer user_data, GDestroyNotify destroy_notify); -void gst_sctp_association_set_on_packet_received (GstSctpAssociation * self, - GstSctpAssociationPacketReceivedCb packet_received_cb, gpointer user_data, GDestroyNotify destroy_notify); + +void gst_sctp_association_set_encoder_ctx (GstSctpAssociation * self, + GstSctpAssociationEncoderCtx * ctx); +void gst_sctp_association_set_decoder_ctx (GstSctpAssociation * self, + GstSctpAssociationDecoderCtx * ctx); + void gst_sctp_association_incoming_packet (GstSctpAssociation * self, const guint8 * buf, guint32 length); GstFlowReturn gst_sctp_association_send_data (GstSctpAssociation * self, From 12fcc25f8b671052c28ea315694e44b56e462f71 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 21 Mar 2023 14:59:40 +0100 Subject: [PATCH 282/339] SCTP: do not assert on the decoders pad, as the association might be disconnecting when on_receive() is called from usrsctplib (cherry picked from commit b50426edac7986eb34c3c732d3e0ebc751ebf70d) --- subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index 89211d7ee02..5699e47b7c3 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -658,7 +658,8 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_ERROR_OBJECT (self, - "The SCTP association must be established before a new stream can be created"); + "The SCTP association must be established before a new stream can be created (state: %d)", + state); return NULL; } @@ -756,7 +757,9 @@ on_receive (GstSctpAssociation * sctp_association, guint8 * buf, GstBuffer *gstbuf; src_pad = get_pad_for_stream_id (self, stream_id); - g_assert (src_pad); + /* If we don't have a src_pad it could mean the association is disconnecting */ + if (!src_pad) + return; GST_DEBUG_OBJECT (src_pad, "Received incoming packet of size %" G_GSIZE_FORMAT From 10fc1f33f3487049a5d4edacbc8ad1052201f7b5 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 23 Mar 2023 10:15:40 +0100 Subject: [PATCH 283/339] SCTP: changes to handle usrsctp bugs Workaround for sctplab/usrsctp#405: - Since the sctp socket can outlive the sctp assoacition, we need to only register/deregister the association when creating/closing the socket. This prevents invalid calls to sctp_packet_out() and receive_cb() in potential invalid states. Workaround for sctplab/usrsctp#383: - Retry usrsctp_finish() for 5seconds. This fixes a race condition between usrsctp_close() and usrsctp_finish() in which a dead socket will be accessed by the SCTP thread. (cherry picked from commit 9cf8ac26e80b60aafd75caa57944df5d03337358) --- .../ext/sctp/sctpassociation.c | 87 ++++++++++++++++--- 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 69dff8dd34b..6489128ec78 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -194,8 +194,6 @@ gst_sctp_association_init (GstSctpAssociation * self) self->use_sock_stream = TRUE; g_mutex_init (&self->association_mutex); - - usrsctp_register_address ((void *) self); } static void @@ -203,13 +201,13 @@ gst_sctp_association_finalize (GObject * object) { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (object); - usrsctp_deregister_address ((void *) self); - /* no need to hold the association_lock, it is held under gst_sctp_association_unref */ g_hash_table_remove (associations_by_id, GUINT_TO_POINTER (self->association_id)); - g_hash_table_remove (ids_by_association, self); + + /* demand we are no longer registered */ + g_assert (!g_hash_table_contains (ids_by_association, self)); g_mutex_clear (&self->association_mutex); @@ -346,6 +344,69 @@ gst_sctp_association_usrsctp_init (void) #endif } +static void +gst_sctp_association_usrsctp_deinit (void) +{ + /* usrsctp_finish could fail, so retry for 5 seconds */ + gint ret; + for (gint i = 0; i < 50; ++i) { + ret = usrsctp_finish (); + if (ret == 0) { + GST_DEBUG ("usrsctp_finish() succeed"); + break; + } + + GST_DEBUG ("usrsctp_finish() failed and returned %d", ret); + g_usleep (100 * G_TIME_SPAN_MILLISECOND); + } + + if (ret != 0) { + GST_WARNING ("usrsctp_finish() failed and returned %d", ret); + } +} + +/* +* Helper register/deregister functions to workaround bug sctplab/usrsctp#405 +* +* The sctp socket can outlive the association, so we need to protect ourselves +* against being called with an invalid reference of GstSctpAssociation. +* To do so, only register/deregister when we create/close the socket in a +* thread-safe way. +* +*/ +static void +gst_sctp_association_register (GstSctpAssociation * self) +{ + G_LOCK (associations_lock); + + /* demand we are not registering twice and we have a valid association_id */ + g_assert (!g_hash_table_contains (ids_by_association, self)); + g_assert (self->association_id != 0); + + g_hash_table_insert (ids_by_association, self, + GUINT_TO_POINTER (self->association_id)); + + G_UNLOCK (associations_lock); + + usrsctp_register_address ((void *) self); +} + +static void +gst_sctp_association_deregister (GstSctpAssociation * self) +{ + usrsctp_deregister_address ((void *) self); + + G_LOCK (associations_lock); + + /* demand we are not deregistering twice and we have a valid association_id */ + g_assert (g_hash_table_contains (ids_by_association, self)); + g_assert (self->association_id != 0); + + g_hash_table_remove (ids_by_association, self); + + G_UNLOCK (associations_lock); +} + /* Public functions */ GstSctpAssociation * @@ -379,8 +440,6 @@ gst_sctp_association_get (guint32 association_id) association_id, NULL); g_hash_table_insert (associations_by_id, GUINT_TO_POINTER (association_id), association); - g_hash_table_insert (ids_by_association, association, - GUINT_TO_POINTER (association_id)); } else { g_object_ref (association); } @@ -406,12 +465,15 @@ gst_sctp_association_unref (GstSctpAssociation * self) g_object_unref (self); if (g_hash_table_size (associations_by_id) == 0) { + /* demand all association have ben deregistered */ + g_assert (g_hash_table_size (ids_by_association) == 0); + g_hash_table_destroy (associations_by_id); g_hash_table_destroy (ids_by_association); associations_by_id = NULL; ids_by_association = NULL; - usrsctp_finish (); + gst_sctp_association_usrsctp_deinit (); } G_UNLOCK (associations_lock); } @@ -629,9 +691,9 @@ static void force_close_unlocked (GstSctpAssociation * self, gboolean change_state) { if (self->sctp_ass_sock) { - struct socket *s = self->sctp_ass_sock; + usrsctp_close (self->sctp_ass_sock); + gst_sctp_association_deregister (self); self->sctp_ass_sock = NULL; - usrsctp_close (s); } self->sctp_assoc_id = 0; @@ -793,6 +855,8 @@ create_sctp_socket (GstSctpAssociation * self) } } + gst_sctp_association_register (self); + return sock; error: if (sock) @@ -904,7 +968,8 @@ association_is_valid (GstSctpAssociation * self) G_LOCK (associations_lock); - valid = g_hash_table_contains (ids_by_association, self); + if (ids_by_association != NULL) + valid = g_hash_table_contains (ids_by_association, self); G_UNLOCK (associations_lock); From 11413bdea0eb5c669ddc5f587395064aec546aec Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 24 Mar 2023 17:39:05 +0100 Subject: [PATCH 284/339] SCTP: removed extra assertion on association id The ID can be zero. --- subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 6489128ec78..94bc5dd868f 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -379,9 +379,8 @@ gst_sctp_association_register (GstSctpAssociation * self) { G_LOCK (associations_lock); - /* demand we are not registering twice and we have a valid association_id */ + /* demand we are not registering twice */ g_assert (!g_hash_table_contains (ids_by_association, self)); - g_assert (self->association_id != 0); g_hash_table_insert (ids_by_association, self, GUINT_TO_POINTER (self->association_id)); @@ -398,9 +397,8 @@ gst_sctp_association_deregister (GstSctpAssociation * self) G_LOCK (associations_lock); - /* demand we are not deregistering twice and we have a valid association_id */ + /* demand we are not deregistering twice */ g_assert (g_hash_table_contains (ids_by_association, self)); - g_assert (self->association_id != 0); g_hash_table_remove (ids_by_association, self); From e96e7f99b98e038378f63226c593f09aaf894247 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 29 Mar 2023 12:35:52 +0200 Subject: [PATCH 285/339] SCTP: do not hold the association mutex when calling GstSctpAssociationPacketReceivedCb Follow a consistent pattern when executing callbacks in handle_message(). --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 94bc5dd868f..12597b4ae3e 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -1239,11 +1239,19 @@ static void handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid) { + GstSctpAssociationPacketReceivedCb callback; + gpointer decoder; + g_mutex_lock (&self->association_mutex); - if (self->decoder_ctx.packet_received_cb) { + callback = self->decoder_ctx.packet_received_cb; + decoder = self->decoder_ctx.element; + if (decoder) + gst_object_ref (decoder); + g_mutex_unlock (&self->association_mutex); + + if (callback) { /* It's the callbacks job to free the data correctly */ - self->decoder_ctx.packet_received_cb (self, data, datalen, stream_id, ppid, - self->decoder_ctx.element); + callback (self, data, datalen, stream_id, ppid, decoder); } else { /* We use this instead of a bare `free()` so that we use the `free` from * the C runtime that usrsctp was built with. This makes a difference on @@ -1251,7 +1259,9 @@ handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, * CRTs. */ usrsctp_freedumpbuffer ((gchar *) data); } - g_mutex_unlock (&self->association_mutex); + + if (decoder) + gst_object_unref (decoder); } static void From 94ef9d7320b31f64bc72e943f4f359deb74caa4f Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 3 Apr 2023 10:41:24 +0200 Subject: [PATCH 286/339] Revert "SCTP: do not hold the association mutex when calling GstSctpAssociationPacketReceivedCb" This reverts commit a120437eba915e5a899308dba2e4850590dee994. We might not need this change after all. --- .../gst-plugins-bad/ext/sctp/sctpassociation.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 12597b4ae3e..94bc5dd868f 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -1239,19 +1239,11 @@ static void handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid) { - GstSctpAssociationPacketReceivedCb callback; - gpointer decoder; - g_mutex_lock (&self->association_mutex); - callback = self->decoder_ctx.packet_received_cb; - decoder = self->decoder_ctx.element; - if (decoder) - gst_object_ref (decoder); - g_mutex_unlock (&self->association_mutex); - - if (callback) { + if (self->decoder_ctx.packet_received_cb) { /* It's the callbacks job to free the data correctly */ - callback (self, data, datalen, stream_id, ppid, decoder); + self->decoder_ctx.packet_received_cb (self, data, datalen, stream_id, ppid, + self->decoder_ctx.element); } else { /* We use this instead of a bare `free()` so that we use the `free` from * the C runtime that usrsctp was built with. This makes a difference on @@ -1259,9 +1251,7 @@ handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, * CRTs. */ usrsctp_freedumpbuffer ((gchar *) data); } - - if (decoder) - gst_object_unref (decoder); + g_mutex_unlock (&self->association_mutex); } static void From 053406ab814233338004aeb1a9190b136d1ae930 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 11 Aug 2023 13:25:04 +0200 Subject: [PATCH 287/339] SCTP: acquire lock in gst_sctp_association_incoming_packet() To fix calling usrsctp_conninput() on a dead socket. user_socketvar.h: added KASSERT to make sure we do not try to reference a dead socket --- .../ext/sctp/sctpassociation.c | 43 +++++++------------ .../sctp/usrsctp/usrsctplib/user_socketvar.h | 1 + 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 94bc5dd868f..bd42722217f 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -124,8 +124,6 @@ static void handle_message (GstSctpAssociation * self, guint8 * data, static void maybe_set_state_to_ready_unlocked (GstSctpAssociation * self); static void gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, GstSctpAssociationState new_state); -static void gst_sctp_association_change_state (GstSctpAssociation * self, - GstSctpAssociationState new_state); static void gst_sctp_association_class_init (GstSctpAssociationClass * klass) @@ -563,7 +561,10 @@ void gst_sctp_association_incoming_packet (GstSctpAssociation * self, const guint8 * buf, guint32 length) { - usrsctp_conninput ((void *) self, (const void *) buf, (size_t) length, 0); + g_mutex_lock (&self->association_mutex); + if (self->sctp_ass_sock) + usrsctp_conninput ((void *) self, (const void *) buf, (size_t) length, 0); + g_mutex_unlock (&self->association_mutex); } GstFlowReturn @@ -998,6 +999,12 @@ receive_cb (struct socket *sock, union sctp_sockstore addr, void *data, return 1; } + + /* If we acquire the lock here it means this is the SCTP timer thread, if + the thread has already acquired the lock, its coming from + gst_sctp_association_incoming_packet() */ + gboolean acquired_lock = g_mutex_trylock (&self->association_mutex); + if (!data) { /* This is a notification that socket shutdown is complete */ GST_INFO_OBJECT (self, "Received shutdown complete notification"); @@ -1018,6 +1025,9 @@ receive_cb (struct socket *sock, union sctp_sockstore addr, void *data, } } + if (acquired_lock) + g_mutex_unlock (&self->association_mutex); + return 1; } @@ -1044,7 +1054,7 @@ handle_notification (GstSctpAssociation * self, break; case SCTP_SHUTDOWN_EVENT: GST_DEBUG_OBJECT (self, "Event: SCTP_SHUTDOWN_EVENT"); - gst_sctp_association_change_state (self, + gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); break; case SCTP_ADAPTATION_INDICATION: @@ -1116,7 +1126,6 @@ handle_sctp_comm_up (GstSctpAssociation * self, const struct sctp_assoc_change *sac) { GST_INFO_OBJECT (self, "SCTP_COMM_UP"); - g_mutex_lock (&self->association_mutex); if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { self->sctp_assoc_id = sac->sac_assoc_id; _apply_aggressive_heartbeat_unlocked (self); @@ -1129,7 +1138,6 @@ handle_sctp_comm_up (GstSctpAssociation * self, GST_WARNING_OBJECT (self, "SCTP association in unexpected state: %d", self->state); } - g_mutex_unlock (&self->association_mutex); } static void @@ -1140,9 +1148,7 @@ handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, sac->sac_state == SCTP_COMM_LOST ? "SCTP_COMM_LOST" : "SCTP_SHUTDOWN_COMP"); - g_mutex_lock (&self->association_mutex); gst_sctp_association_disconnect_unlocked (self, FALSE); - g_mutex_unlock (&self->association_mutex); } static void @@ -1151,20 +1157,16 @@ gst_sctp_association_notify_restart (GstSctpAssociation * self) GstSctpAssociationRestartCb restart_cb; gpointer user_data; - g_mutex_lock (&self->association_mutex); restart_cb = self->decoder_ctx.restart_cb; user_data = self->decoder_ctx.element; if (user_data) gst_object_ref (user_data); - g_mutex_unlock (&self->association_mutex); if (restart_cb) restart_cb (self, user_data); - g_mutex_lock (&self->association_mutex); if (user_data) gst_object_unref (user_data); - g_mutex_unlock (&self->association_mutex); } static void @@ -1174,20 +1176,16 @@ gst_sctp_association_notify_stream_reset (GstSctpAssociation * self, GstSctpAssociationStreamResetCb stream_reset_cb; gpointer user_data; - g_mutex_lock (&self->association_mutex); stream_reset_cb = self->decoder_ctx.stream_reset_cb; user_data = self->decoder_ctx.element; if (user_data) gst_object_ref (user_data); - g_mutex_unlock (&self->association_mutex); if (stream_reset_cb) stream_reset_cb (self, stream_id, user_data); - g_mutex_lock (&self->association_mutex); if (user_data) gst_object_unref (user_data); - g_mutex_unlock (&self->association_mutex); } static void @@ -1211,7 +1209,7 @@ handle_association_changed (GstSctpAssociation * self, break; case SCTP_CANT_STR_ASSOC: GST_WARNING_OBJECT (self, "SCTP event SCTP_CANT_STR_ASSOC received"); - gst_sctp_association_change_state (self, + gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_ERROR); break; } @@ -1239,7 +1237,6 @@ static void handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid) { - g_mutex_lock (&self->association_mutex); if (self->decoder_ctx.packet_received_cb) { /* It's the callbacks job to free the data correctly */ self->decoder_ctx.packet_received_cb (self, data, datalen, stream_id, ppid, @@ -1251,7 +1248,6 @@ handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, * CRTs. */ usrsctp_freedumpbuffer ((gchar *) data); } - g_mutex_unlock (&self->association_mutex); } static void @@ -1291,12 +1287,3 @@ gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, gst_object_unref (encoder); gst_sctp_association_unref (self); } - -static void -gst_sctp_association_change_state (GstSctpAssociation * self, - GstSctpAssociationState new_state) -{ - g_mutex_lock (&self->association_mutex); - gst_sctp_association_change_state_unlocked (self, new_state); - g_mutex_unlock (&self->association_mutex); -} diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socketvar.h b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socketvar.h index 6b79b5908c0..6e7870260c5 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socketvar.h +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socketvar.h @@ -403,6 +403,7 @@ void sofree(struct socket *so); #define soref(so) do { \ SOCK_LOCK_ASSERT(so); \ + KASSERT((so)->so_count > 0, ("soref")); \ ++(so)->so_count; \ } while (0) From 65f313c6b44c6be7f695758cbb458fc1bf9b33f8 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 22 Jun 2023 14:43:02 +0200 Subject: [PATCH 288/339] SCTP: removing usrsctp_shutdown() call, usrsctp_close() should have the same effect --- .../ext/sctp/sctpassociation.c | 22 +++---------------- .../ext/sctp/sctpassociation.h | 1 - 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index bd42722217f..8e21186b102 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -704,26 +704,11 @@ force_close_unlocked (GstSctpAssociation * self, gboolean change_state) } static void -gst_sctp_association_disconnect_unlocked (GstSctpAssociation * self, - gboolean try_shutdown) +gst_sctp_association_disconnect_unlocked (GstSctpAssociation * self) { if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { gst_sctp_association_change_state_unlocked (self, GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); - - if (try_shutdown && self->use_sock_stream && self->sctp_ass_sock) { - GST_INFO_OBJECT (self, "SCTP association shutting down"); - self->shutdown = FALSE; - if (usrsctp_shutdown (self->sctp_ass_sock, SHUT_RDWR) == 0) { - /* wait for shutdown to complete */ - guint cs_to_wait = 100; /* 1s */ - while (!self->shutdown && cs_to_wait > 0) { - g_usleep (G_USEC_PER_SEC / 100); - cs_to_wait--; - } - self->shutdown = FALSE; - } - } } /* Fall through to ensure the transition to disconnected occurs */ @@ -739,7 +724,7 @@ void gst_sctp_association_disconnect (GstSctpAssociation * self) { g_mutex_lock (&self->association_mutex); - gst_sctp_association_disconnect_unlocked (self, TRUE); + gst_sctp_association_disconnect_unlocked (self); g_mutex_unlock (&self->association_mutex); } @@ -1008,7 +993,6 @@ receive_cb (struct socket *sock, union sctp_sockstore addr, void *data, if (!data) { /* This is a notification that socket shutdown is complete */ GST_INFO_OBJECT (self, "Received shutdown complete notification"); - self->shutdown = TRUE; } else { if (flags & MSG_NOTIFICATION) { handle_notification (self, (const union sctp_notification *) data, @@ -1148,7 +1132,7 @@ handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, sac->sac_state == SCTP_COMM_LOST ? "SCTP_COMM_LOST" : "SCTP_SHUTDOWN_COMP"); - gst_sctp_association_disconnect_unlocked (self, FALSE); + gst_sctp_association_disconnect_unlocked (self); } static void diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 3779a6d4829..d618f248384 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -104,7 +104,6 @@ struct _GstSctpAssociation gboolean use_sock_stream; gboolean aggressive_heartbeat; struct socket *sctp_ass_sock; - gboolean shutdown; GMutex association_mutex; From 1a7dab16ee0e385fd6454abddc43be5a9b4f5781 Mon Sep 17 00:00:00 2001 From: Will Miller Date: Fri, 21 Apr 2023 16:34:47 +0100 Subject: [PATCH 289/339] rtpopuspay: pass buffer offset to base_payload_push When using perfect-rtptime, rtpbasepayload needs to have access to buffer offsets to calculate the RTP timestamps. (cherry picked from commit 064bfba9b0c758d2f5fbbaf52810da47e8d88079) (cherry picked from commit c2106acc629344dbd2800f9a05886d658156b290) --- subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c index e62868c3767..840457412d8 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c @@ -328,6 +328,7 @@ gst_rtp_opus_pay_handle_buffer (GstRTPBasePayload * basepayload, GstRtpOPUSPay *self = GST_RTP_OPUS_PAY_CAST (basepayload); GstBuffer *outbuf; GstClockTime pts, dts, duration; + guint64 offset; /* DTX packets are zero-length frames, with a 1 or 2-bytes header */ if (self->dtx && gst_buffer_get_size (buffer) <= 2) { @@ -341,6 +342,7 @@ gst_rtp_opus_pay_handle_buffer (GstRTPBasePayload * basepayload, pts = GST_BUFFER_PTS (buffer); dts = GST_BUFFER_DTS (buffer); duration = GST_BUFFER_DURATION (buffer); + offset = GST_BUFFER_OFFSET (buffer); outbuf = gst_rtp_base_payload_allocate_output_buffer (basepayload, 0, 0, 0); @@ -351,6 +353,7 @@ gst_rtp_opus_pay_handle_buffer (GstRTPBasePayload * basepayload, GST_BUFFER_PTS (outbuf) = pts; GST_BUFFER_DTS (outbuf) = dts; GST_BUFFER_DURATION (outbuf) = duration; + GST_BUFFER_OFFSET (outbuf) = offset; if (self->marker) { GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; From 8af883b4924804ebf0a31ba7177e94b5b98ba15c Mon Sep 17 00:00:00 2001 From: Will Miller Date: Fri, 21 Apr 2023 17:28:10 +0100 Subject: [PATCH 290/339] rtpbasedepayload: set default buffer offset based on RTP timestamp Set a default offset on buffers: the delta between this buffer's RTP timestamp and the first RTP timestamp. This can be used when forwarding (depayloading/repayloading) media for codecs without specific offset logic in their payloader subclasses (e.g. Opus). The default offset allows the symmetric payloader to reconstruct the original RTP timestamp deltas for its output RTP timestamps, rather than having to rely on the (potentially altered) buffer PTS. (cherry picked from commit a97f44f52ba24220bcc2251fa40c6c1a638d03af) (cherry picked from commit 52edb23d02006c8e9fd817b00894fe74922255d1) --- .../gst-libs/gst/rtp/gstrtpbasedepayload.c | 18 ++++++++- .../tests/check/libs/rtpbasedepayload.c | 37 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c index d683f6d12de..4aadd592e56 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c @@ -84,9 +84,10 @@ struct _GstRTPBaseDepayloadPrivate GstClockTime pts; GstClockTime dts; GstClockTime duration; + guint64 offset; GstClockTime ref_ts; - + guint64 first_rtptime; guint32 last_ssrc; guint32 last_seqnum; guint32 last_rtptime; @@ -420,6 +421,8 @@ gst_rtp_base_depayload_class_init (GstRTPBaseDepayloadClass * klass) "Base class for RTP Depayloaders"); } +#define RTPTIME_NONE G_MAXUINT64 + static void gst_rtp_base_depayload_init (GstRTPBaseDepayload * filter, GstRTPBaseDepayloadClass * klass) @@ -461,6 +464,8 @@ gst_rtp_base_depayload_init (GstRTPBaseDepayload * filter, priv->pts = -1; priv->duration = -1; priv->ref_ts = -1; + priv->first_rtptime = RTPTIME_NONE; + priv->offset = GST_BUFFER_OFFSET_NONE; priv->source_info = DEFAULT_SOURCE_INFO; priv->ignore_gaps = DEFAULT_IGNORE_GAPS; priv->max_reorder = DEFAULT_MAX_REORDER; @@ -789,6 +794,12 @@ gst_rtp_base_depayload_handle_buffer (GstRTPBaseDepayload * filter, seqnum = gst_rtp_buffer_get_seq (&rtp); rtptime = gst_rtp_buffer_get_timestamp (&rtp); + if (priv->first_rtptime == RTPTIME_NONE) { + // First buffer handled + priv->first_rtptime = rtptime; + } + priv->offset = rtptime - priv->first_rtptime; + priv->last_seqnum = seqnum; priv->last_rtptime = rtptime; @@ -1415,6 +1426,9 @@ gst_rtp_base_depayload_set_headers (GstRTPBaseDepayload * depayload, GST_BUFFER_DTS (buffer) = priv->dts; if (!GST_CLOCK_TIME_IS_VALID (duration)) GST_BUFFER_DURATION (buffer) = priv->duration; + if (!GST_BUFFER_OFFSET_IS_VALID (buffer)) { + GST_BUFFER_OFFSET (buffer) = priv->offset; + } if (G_UNLIKELY (depayload->priv->discont)) { GST_LOG_OBJECT (depayload, "Marking DISCONT on output buffer"); @@ -1426,6 +1440,7 @@ gst_rtp_base_depayload_set_headers (GstRTPBaseDepayload * depayload, priv->pts = GST_CLOCK_TIME_NONE; priv->dts = GST_CLOCK_TIME_NONE; priv->duration = GST_CLOCK_TIME_NONE; + priv->offset = GST_BUFFER_OFFSET_NONE; if (priv->input_buffer) { if (priv->source_info) @@ -1705,6 +1720,7 @@ gst_rtp_base_depayload_change_state (GstElement * element, priv->negotiated = FALSE; priv->discont = FALSE; priv->segment_seqnum = GST_SEQNUM_INVALID; + priv->first_rtptime = RTPTIME_NONE; break; case GST_STATE_CHANGE_PAUSED_TO_PLAYING: break; diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c b/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c index d006acc6360..35539d87fbf 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c @@ -2084,6 +2084,39 @@ GST_START_TEST (rtp_base_depayload_hdr_ext_aggregate_flush) GST_END_TEST; +GST_START_TEST (rtp_base_depayload_rtptime_buffer_offset) +{ + State *state; + state = create_depayloader ("application/x-rtp", NULL); + set_state (state, GST_STATE_PLAYING); + gint buf_idx = 0; + + /* buffer offset should start at 0 */ + push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, + "rtptime", 1000, "seq", 0x4242 + buf_idx, + NULL); + validate_buffer (buf_idx++, "offset", 0, NULL); + + /* buffer offset should be the delta between this rtptime and first + * rtptime */ + push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, + "rtptime", 1500, "seq", 0x4242 + buf_idx, + NULL); + validate_buffer (buf_idx++, "offset", 500, NULL); + + /* buffer offset already set so should not be modified */ + push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, + "rtptime", 2000, "seq", 0x4242 + buf_idx, + "offset", 5678, + NULL); + validate_buffer (buf_idx++, "offset", 5678, NULL); + + set_state (state, GST_STATE_NULL); + destroy_depayloader (state); +} + +GST_END_TEST; + static Suite * rtp_basepayloading_suite (void) { @@ -2127,11 +2160,15 @@ rtp_basepayloading_suite (void) tcase_add_test (tc_chain, rtp_base_depayload_clear_extensions); tcase_add_test (tc_chain, rtp_base_depayload_multiple_exts); tcase_add_test (tc_chain, rtp_base_depayload_caps_request_ignored); + tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_caps_change); tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_aggregate); tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_aggregate_drop); tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_aggregate_delayed); tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_aggregate_flush); + + tcase_add_test (tc_chain, rtp_base_depayload_rtptime_buffer_offset); + return s; } From 3df6fbdd765363162d50f3d6c8de13247f1e5e6d Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Thu, 4 May 2023 23:29:30 +0900 Subject: [PATCH 291/339] rtpbasedepayload: test: correct order of fields The gist of the issue and fix are somewhat obscure and relate with how a buffer is mapped into before on different platforms. The test uses a va_args method to create, set and push an RTP buffer to the depayloader. However, due to the nature of the fields that can be set, and the use of the va_args. The ordering of this fields matter. This is because some of this fields are only set of a GstRTPBuffer and other are public GstBuffer properties. The problem lies when we set a GstRTPBuffer field, which requires a map of the buffer and **then** try to set a public property on the raw GstBuffer that has already been mapped! Fix this by: 1). correct the order of fields set on the buffer. 2). assert hard when trying to set a public property on the raw GstBuffer after it has been mapped. (cherry picked from commit 38571dbd0ba833d60b6fca0f3b87fd620d47f3e7) --- .../tests/check/libs/rtpbasedepayload.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c b/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c index 35539d87fbf..f802ebbccd7 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c @@ -405,12 +405,15 @@ rtp_buffer_set_valist (GstBuffer * buf, const gchar * field, va_list var_args, while (field) { if (!g_strcmp0 (field, "pts")) { + fail_unless (!mapped); GstClockTime pts = va_arg (var_args, GstClockTime); GST_BUFFER_PTS (buf) = pts; } else if (!g_strcmp0 (field, "offset")) { + fail_unless (!mapped); guint64 offset = va_arg (var_args, guint64); GST_BUFFER_OFFSET (buf) = offset; } else if (!g_strcmp0 (field, "discont")) { + fail_unless (!mapped); gboolean discont = va_arg (var_args, gboolean); if (discont) { GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DISCONT); @@ -1101,10 +1104,10 @@ GST_START_TEST (rtp_base_depayload_seq_discont_test) "rtptime", G_GUINT64_CONSTANT (0x1234), "seq", 1, NULL); push_rtp_buffer (state, - "extra-ref", TRUE, "pts", 2 * GST_SECOND, "rtptime", G_GUINT64_CONSTANT (0x1234) + DEFAULT_CLOCK_RATE / 2, - "seq", 33333, NULL); + "seq", 33333, + "extra-ref", TRUE, NULL); set_state (state, GST_STATE_NULL); @@ -2093,22 +2096,18 @@ GST_START_TEST (rtp_base_depayload_rtptime_buffer_offset) /* buffer offset should start at 0 */ push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, - "rtptime", 1000, "seq", 0x4242 + buf_idx, - NULL); + "rtptime", 1000, "seq", 0x4242 + buf_idx, NULL); validate_buffer (buf_idx++, "offset", 0, NULL); /* buffer offset should be the delta between this rtptime and first * rtptime */ push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, - "rtptime", 1500, "seq", 0x4242 + buf_idx, - NULL); + "rtptime", 1500, "seq", 0x4242 + buf_idx, NULL); validate_buffer (buf_idx++, "offset", 500, NULL); /* buffer offset already set so should not be modified */ - push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, - "rtptime", 2000, "seq", 0x4242 + buf_idx, - "offset", 5678, - NULL); + push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, "offset", 5678, + "rtptime", 2000, "seq", 0x4242 + buf_idx, NULL); validate_buffer (buf_idx++, "offset", 5678, NULL); set_state (state, GST_STATE_NULL); From b60aec99d5e7f3b8870b179d4c936d63c15dec7e Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Wed, 24 May 2023 12:50:11 +0200 Subject: [PATCH 292/339] vp8pay: dont store partition_size --- subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c | 10 ++++++---- subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c index c216a125cfa..2fe848be825 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c @@ -410,16 +410,18 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, pdata = data + partition0_size; /* Set up mapping */ + guint partition_size[9]; + self->n_partitions = partitions + 1; self->partition_offset[0] = 0; - self->partition_size[0] = partition0_size + (partitions - 1) * 3; + partition_size[0] = partition0_size + (partitions - 1) * 3; - self->partition_offset[1] = self->partition_size[0]; + self->partition_offset[1] = partition_size[0]; for (i = 1; i < partitions; i++) { guint psize = (pdata[2] << 16 | pdata[1] << 8 | pdata[0]); pdata += 3; - self->partition_size[i] = psize; + partition_size[i] = psize; self->partition_offset[i + 1] = self->partition_offset[i] + psize; } @@ -428,7 +430,7 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, if (self->partition_offset[i] >= size) goto error; - self->partition_size[i] = size - self->partition_offset[i]; + partition_size[i] = size - self->partition_offset[i]; self->partition_offset[i + 1] = size; diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h index 748bec16501..1a5cc765e7c 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h @@ -57,7 +57,7 @@ struct _GstRtpVP8Pay GstRTPBasePayload parent; gboolean is_keyframe; gint n_partitions; - /* Treat frame header & tag & partition size block as the first partition, + /* Treat frame header & tag as the first partition, * folowed by max. 8 data partitions. last offset is the end of the buffer */ guint partition_offset[10]; guint partition_size[9]; From 8aa1eb4c76d47bf57662d667cd820fc8c5101b15 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Wed, 24 May 2023 13:16:22 +0200 Subject: [PATCH 293/339] vp8pay: introduce new property parse-frame Controls weather or not we would want to parse the bitstream in order to extract partition information, which is currently only used if no custom meta is present. --- .../gst-plugins-good/gst/rtp/gstrtpvp8pay.c | 26 +++++++++++++++---- .../gst-plugins-good/gst/rtp/gstrtpvp8pay.h | 2 ++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c index 2fe848be825..2213c20db4d 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c @@ -41,13 +41,15 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_vp8_pay_debug); #define DEFAULT_PICTURE_ID_MODE VP8_PAY_NO_PICTURE_ID #define DEFAULT_PICTURE_ID_OFFSET (-1) +#define DEFAULT_PARSE_FRAMES TRUE enum { PROP_0, PROP_PICTURE_ID, PROP_PICTURE_ID_MODE, - PROP_PICTURE_ID_OFFSET + PROP_PICTURE_ID_OFFSET, + PROP_PARSE_FRAMES, }; #define GST_TYPE_RTP_VP8_PAY_PICTURE_ID_MODE (gst_rtp_vp8_pay_picture_id_mode_get_type()) @@ -193,6 +195,12 @@ gst_rtp_vp8_pay_class_init (GstRtpVP8PayClass * gst_rtp_vp8_pay_class) GST_TYPE_RTP_VP8_PAY_PICTURE_ID_MODE, DEFAULT_PICTURE_ID_MODE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PARSE_FRAMES, + g_param_spec_boolean ("parse-frames", "Parse frames", + "Whether to parse/validate frame bitstreams", + DEFAULT_PARSE_FRAMES, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT_ONLY)); + /** * rtpvp8pay:picture-id-offset: * @@ -240,6 +248,9 @@ gst_rtp_vp8_pay_set_property (GObject * object, rtpvp8pay->picture_id_offset = g_value_get_int (value); gst_rtp_vp8_pay_picture_id_reset (rtpvp8pay); break; + case PROP_PARSE_FRAMES: + rtpvp8pay->parse_frames = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -262,6 +273,9 @@ gst_rtp_vp8_pay_get_property (GObject * object, case PROP_PICTURE_ID_OFFSET: g_value_set_int (value, rtpvp8pay->picture_id_offset); break; + case PROP_PARSE_FRAMES: + g_value_set_boolean (value, rtpvp8pay->parse_frames); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -677,10 +691,12 @@ gst_rtp_vp8_pay_handle_buffer (GstRTPBasePayload * payload, GstBuffer * buffer) meta = gst_buffer_get_custom_meta (buffer, "GstVP8Meta"); delta_unit = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT); - if (G_UNLIKELY (!gst_rtp_vp8_pay_parse_frame (self, buffer, size))) { - GST_ELEMENT_ERROR (self, STREAM, ENCODE, (NULL), - ("Failed to parse VP8 frame")); - return GST_FLOW_ERROR; + if (self->parse_frames) { + if (G_UNLIKELY (!gst_rtp_vp8_pay_parse_frame (self, buffer, size))) { + GST_ELEMENT_ERROR (self, STREAM, ENCODE, (NULL), + ("Failed to parse VP8 frame")); + return GST_FLOW_ERROR; + } } if (meta) { diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h index 1a5cc765e7c..132fc0cdfe4 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h @@ -1,3 +1,4 @@ +/* /* * gstrtpvp8pay.h - Header for GstRtpVP8Pay * Copyright (C) 2011 Sjoerd Simons @@ -55,6 +56,7 @@ struct _GstRtpVP8PayClass struct _GstRtpVP8Pay { GstRTPBasePayload parent; + gboolean parse_frames; gboolean is_keyframe; gint n_partitions; /* Treat frame header & tag as the first partition, From 6950986d70405350013c15b12d24cf058a913bca Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Wed, 24 May 2023 13:18:18 +0200 Subject: [PATCH 294/339] vp8pay: add additional comments around bitstream parsing --- .../gst-plugins-good/gst/rtp/gstrtpvp8pay.c | 27 ++++++++++++++++++- subprojects/gstreamer/gst/gstinfo.c | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c index 2213c20db4d..dcc436bbdb1 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c @@ -311,6 +311,19 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, gst_bit_reader_init (&reader, data, size); + /* from RFC 6386 Section 9.1: Uncompressed data chunk + * + * 0 1 2 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |F|Vers.|S| Partition size | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * Frame type: 0 key-frame, 1 inter-frame + * Version (0-3) + * Show_frame + * First partition size + * + */ self->is_keyframe = keyframe = ((data[0] & 0x1) == 0); version = (data[0] >> 1) & 0x7; @@ -330,6 +343,16 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, goto error; if (keyframe) { + /* + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | 0x9d012a | Scale & Width | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | | Scale & Height | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + */ /* check start tag: 0x9d 0x01 0x2a */ if (!gst_bit_reader_get_bits_uint8 (&reader, &tmp8, 8) || tmp8 != 0x9d) goto error; @@ -340,7 +363,9 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, if (!gst_bit_reader_get_bits_uint8 (&reader, &tmp8, 8) || tmp8 != 0x2a) goto error; - /* Skip horizontal size code (16 bits) vertical size code (16 bits) */ + /* Skip horizontal size code (16 bits) vertical size code (16 bits) + * (2 bits horizonal scaling << 14 | Width) + * (2 bits vertical scaling << 14 | Height) */ if (!gst_bit_reader_skip (&reader, 32)) goto error; } diff --git a/subprojects/gstreamer/gst/gstinfo.c b/subprojects/gstreamer/gst/gstinfo.c index 7515e065ad9..6410b86bc97 100644 --- a/subprojects/gstreamer/gst/gstinfo.c +++ b/subprojects/gstreamer/gst/gstinfo.c @@ -3512,7 +3512,7 @@ gst_debug_print_stack_trace (void) G_LOCK (win_print_mutex); #endif - g_print ("%s\n", trace); + g_printerr ("%s\n", trace); #ifdef G_OS_WIN32 G_UNLOCK (win_print_mutex); From d8ca5042aa70ff93d1056f6ca440365f14a04428 Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Wed, 24 May 2023 13:23:44 +0200 Subject: [PATCH 295/339] vp8pay: use DELTA_UNIT buffer flag know a frame type --- subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c index dcc436bbdb1..26a83e990fb 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c @@ -324,7 +324,7 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, * First partition size * */ - self->is_keyframe = keyframe = ((data[0] & 0x1) == 0); + keyframe = ((data[0] & 0x1) == 0); version = (data[0] >> 1) & 0x7; if (G_UNLIKELY (version > 3)) { From 374c8b22ba57a71b66d1008a08daeaaaa07fa291 Mon Sep 17 00:00:00 2001 From: Will Miller Date: Wed, 24 May 2023 12:16:51 +0200 Subject: [PATCH 296/339] rtpvp9pay: add parse-frames property --- .../gst-plugins-good/gst/rtp/gstrtpvp9pay.c | 35 ++++++++++++++++--- .../gst-plugins-good/gst/rtp/gstrtpvp9pay.h | 1 + 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c index f5db16b2cf2..f17cc91a71b 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c @@ -42,6 +42,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_vp9_pay_debug); #define DEFAULT_PICTURE_ID_MODE VP9_PAY_NO_PICTURE_ID #define DEFAULT_PICTURE_ID_OFFSET (-1) +#define DEFAULT_PARSE_FRAMES TRUE enum { @@ -49,6 +50,7 @@ enum PROP_PICTURE_ID, PROP_PICTURE_ID_MODE, PROP_PICTURE_ID_OFFSET, + PROP_PARSE_FRAMES, }; #define GST_TYPE_RTP_VP9_PAY_PICTURE_ID_MODE (gst_rtp_vp9_pay_picture_id_mode_get_type()) @@ -136,6 +138,7 @@ gst_rtp_vp9_pay_init (GstRtpVP9Pay * obj) { obj->picture_id_mode = DEFAULT_PICTURE_ID_MODE; obj->picture_id_offset = DEFAULT_PICTURE_ID_OFFSET; + obj->parse_frames = DEFAULT_PARSE_FRAMES; gst_rtp_vp9_pay_picture_id_reset (obj); } @@ -181,6 +184,12 @@ gst_rtp_vp9_pay_class_init (GstRtpVP9PayClass * gst_rtp_vp9_pay_class) -1, 0x7FFF, DEFAULT_PICTURE_ID_OFFSET, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PARSE_FRAMES, + g_param_spec_boolean ("parse-frames", "Parse frames", + "Whether to parse/validate frame bitstreams", + DEFAULT_PARSE_FRAMES, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT_ONLY)); + gst_element_class_add_static_pad_template (element_class, &gst_rtp_vp9_pay_sink_template); gst_element_class_add_static_pad_template (element_class, @@ -215,6 +224,9 @@ gst_rtp_vp9_pay_set_property (GObject * object, rtpvp9pay->picture_id_offset = g_value_get_int (value); gst_rtp_vp9_pay_picture_id_reset (rtpvp9pay); break; + case PROP_PARSE_FRAMES: + rtpvp9pay->parse_frames = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -237,6 +249,9 @@ gst_rtp_vp9_pay_get_property (GObject * object, case PROP_PICTURE_ID_OFFSET: g_value_set_int (value, rtpvp9pay->picture_id_offset); break; + case PROP_PARSE_FRAMES: + g_value_set_boolean (value, rtpvp9pay->parse_frames); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -459,9 +474,12 @@ gst_rtp_vp9_create_header_buffer (GstRtpVP9Pay * self, hdrlen, 0, 0); gst_rtp_buffer_map (out, GST_MAP_READWRITE, &rtpbuffer); p = gst_rtp_buffer_get_payload (&rtpbuffer); + + /* Zero-initialise required first byte*/ p[0] = 0x0; if (self->picture_id_mode != VP9_PAY_NO_PICTURE_ID) { + /* I=1 */ p[0] |= 0x80; if (self->picture_id_mode == VP9_PAY_PICTURE_ID_7BITS) { /* M=0 */ @@ -474,13 +492,17 @@ gst_rtp_vp9_create_header_buffer (GstRtpVP9Pay * self, } if (!self->is_keyframe) + /* P=1 */ p[0] |= 0x40; if (start) + /* B=1 */ p[0] |= 0x08; if (mark) + /* E=1 */ p[0] |= 0x04; if (self->is_keyframe && start) { + /* V=1 */ p[0] |= 0x02; /* scalability structure, hard coded for now to be similar to chromium for * quick and dirty interop */ @@ -570,12 +592,17 @@ gst_rtp_vp9_pay_handle_buffer (GstRTPBasePayload * payload, GstBuffer * buffer) size = gst_buffer_get_size (buffer); delta_unit = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT); - if (G_UNLIKELY (!gst_rtp_vp9_pay_parse_frame (self, buffer, size))) { - GST_ELEMENT_ERROR (self, STREAM, ENCODE, (NULL), - ("Failed to parse VP9 frame")); - return GST_FLOW_ERROR; + if (self->parse_frames) { + if (G_UNLIKELY (!gst_rtp_vp9_pay_parse_frame (self, buffer, size))) { + GST_ELEMENT_ERROR (self, STREAM, ENCODE, (NULL), + ("Failed to parse VP9 frame")); + return GST_FLOW_ERROR; + } + } else { + self->is_keyframe = !GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT); } + mtu = GST_RTP_BASE_PAYLOAD_MTU (payload); vp9_hdr_len = gst_rtp_vp9_calc_header_len (self, TRUE); max_paylen = gst_rtp_buffer_calc_payload_len (mtu - vp9_hdr_len, 0, 0); diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.h index b2ad88445f9..eef054df58a 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.h @@ -62,6 +62,7 @@ struct _GstRtpVP9Pay GstVP9RtpPayPictureIDMode picture_id_mode; gint picture_id_offset; gint picture_id; + gboolean parse_frames; }; GType gst_rtp_vp9_pay_get_type (void); From 88699e5812a2aa09e3b8bcf6ab58beeb9e6329f7 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 31 Jan 2023 10:43:22 +0100 Subject: [PATCH 297/339] check: compile libcompat when building statically ...or else we have to carry rt_lib as a dependency --- subprojects/gstreamer/libs/gst/check/libcheck/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gstreamer/libs/gst/check/libcheck/meson.build b/subprojects/gstreamer/libs/gst/check/libcheck/meson.build index af6210b0a05..2eb8659d9c9 100644 --- a/subprojects/gstreamer/libs/gst/check/libcheck/meson.build +++ b/subprojects/gstreamer/libs/gst/check/libcheck/meson.build @@ -40,7 +40,7 @@ if not cdata.has('HAVE_GETLINE') endif # FIXME: check that timer_create, timer_settime, timer_delete are in rt_lib -if not rt_lib.found() +if not rt_lib.found() or get_option('default_library') == 'static' libcheck_files += files( 'libcompat/timer_create.c', 'libcompat/timer_settime.c', From cc150b0df47357e090bdb29fca2c0e5b8ec77fec Mon Sep 17 00:00:00 2001 From: Misha Baranov Date: Wed, 8 Mar 2023 18:09:19 +0100 Subject: [PATCH 298/339] ahc2src: add proper GstVideoMeta to the buffers --- .../sys/androidmedia/gstahc2src.c | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c index b62af9c7983..eefea40fbc9 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c @@ -1478,6 +1478,11 @@ gst_ahc2_src_camera_close (GstAHC2Src * self) { GST_DEBUG_OBJECT (self, "Closing Camera"); + if (self->image_reader) { + AImageReader_setImageListener (self->image_reader, NULL); + g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); + } + g_clear_pointer (&self->capture_sout, (GDestroyNotify) ACaptureSessionOutput_free); @@ -1819,12 +1824,6 @@ _aimage_to_gstbuffer (GstAHC2Src * self, AImage * image, GstBuffer * buffer) AImage_getPlanePixelStride (image, 1, &u_pixelstride); AImage_getPlanePixelStride (image, 2, &v_pixelstride); - if (y_stride != self->width) { - GST_ERROR_OBJECT (self, - "Assumption about AImage being stride=width broken!"); - g_assert_not_reached (); - } - wrapped_aimage = g_new0 (GstWrappedAImage, 1); wrapped_aimage->refcount = 1; wrapped_aimage->ahc2src = g_object_ref (self); @@ -1837,28 +1836,49 @@ _aimage_to_gstbuffer (GstAHC2Src * self, AImage * image, GstBuffer * buffer) (GDestroyNotify) gst_wrapped_aimage_unref); gst_buffer_append_memory (buffer, y_mem); - /* if all of this is true, we can safly assume a NV12 layout, where the whole memory for the UV plane is contained inside both u_data and v_data, with only 1 pixel moved */ if (u_length == v_length && /* we have the same length */ u_stride == v_stride && /* we have the same stride */ ((u_data + 1 == v_data) || (v_data + 1 == u_data)) && /* v_data is basically pointing one pixel into u_data (or v.v), otherwise identical, expected for NV12/NV21 */ - u_pixelstride == 2 && v_pixelstride == 2 && /* the packing you expect from inteleaved U/V/U/V in NV12 */ - u_length == y_length / u_pixelstride - 1) { /* when reading all the U pixels, you expect the length to be /2-1 */ + u_pixelstride == 2 && v_pixelstride == 2) { /* the packing you expect from inteleaved U/V/U/V in NV12 */ + + gsize offset[GST_VIDEO_MAX_PLANES] = { 0, y_length, 0, 0 }; + gint stride[GST_VIDEO_MAX_PLANES] = { y_stride, u_stride, 0, 0 }; + GstVideoAlignment align; + gst_video_alignment_reset (&align); + align.padding_right = y_stride - self->width; uv_mem = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, u_data, y_length / 2, 0, y_length / 2, gst_wrapped_aimage_ref (wrapped_aimage), (GDestroyNotify) gst_wrapped_aimage_unref); gst_buffer_append_memory (buffer, uv_mem); + + gst_video_meta_set_alignment (gst_buffer_add_video_meta_full (buffer, + GST_VIDEO_FRAME_FLAG_NONE, GST_VIDEO_FORMAT_NV12, self->width, + self->height, 2, offset, stride), align); + + GST_INFO_OBJECT (self, + "y_length=%d u_length=%d, v_length=%d, " + "y_stride=%d, u_stride=%d, v_stride=%d, " + "y_data=%p, u_data=%p, v_data=%p, " + "width=%d, height=%d, " + "u_pixelstride=%d, v_pixelstride=%d", + y_length, u_length, v_length, y_stride, u_stride, v_stride, + y_data, u_data, v_data, self->width, self->height, + u_pixelstride, v_pixelstride); } else { /* we need this assumption to be true for now! */ GST_ERROR_OBJECT (self, "Not getting the NV12 memory-layout we expected: " "y_length=%d u_length=%d, v_length=%d, " - "u_stride=%d, v_stride=%d, u_data=%p, v_data=%p, " + "y_stride=%d, u_stride=%d, v_stride=%d, " + "y_data=%p, u_data=%p, v_data=%p, " + "width=%d, height=%d, " "u_pixelstride=%d, v_pixelstride=%d", - y_length, u_length, v_length, u_stride, v_stride, u_data, v_data, + y_length, u_length, v_length, y_stride, u_stride, v_stride, + y_data, u_data, v_data, self->width, self->height, u_pixelstride, v_pixelstride); g_assert_not_reached (); } From fe72f6a19f91d442e593bb13476bb896ada8b5ae Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 20 Mar 2023 14:48:58 +0100 Subject: [PATCH 299/339] glshader: unlock before notify, and use dispose instead of finalize To avoid deadlock when shutting down. --- .../gst-plugins-base/gst-libs/gst/gl/gstglshader.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglshader.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglshader.c index 09bdda4321a..05509fab4ae 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglshader.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglshader.c @@ -113,7 +113,7 @@ _cleanup_shader (GstGLContext * context, GstGLShader * shader) } static void -gst_gl_shader_finalize (GObject * object) +gst_gl_shader_dispose (GObject * object) { GstGLShader *shader; GstGLShaderPrivate *priv; @@ -134,7 +134,7 @@ gst_gl_shader_finalize (GObject * object) shader->context = NULL; } - G_OBJECT_CLASS (gst_gl_shader_parent_class)->finalize (object); + G_OBJECT_CLASS (gst_gl_shader_parent_class)->dispose (object); } static void @@ -171,7 +171,7 @@ gst_gl_shader_class_init (GstGLShaderClass * klass) /* bind class methods .. */ GObjectClass *obj_class = G_OBJECT_CLASS (klass); - obj_class->finalize = gst_gl_shader_finalize; + obj_class->dispose = gst_gl_shader_dispose; obj_class->set_property = gst_gl_shader_set_property; obj_class->get_property = gst_gl_shader_get_property; @@ -755,7 +755,9 @@ gst_gl_shader_release_unlocked (GstGLShader * shader) priv->linked = FALSE; g_hash_table_remove_all (priv->uniform_locations); + GST_OBJECT_UNLOCK (shader); g_object_notify (G_OBJECT (shader), "linked"); + GST_OBJECT_LOCK (shader); } /** From 47fbedc0c8774fa937c3c1080c89835efe94582e Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Mon, 20 Mar 2023 14:50:36 +0100 Subject: [PATCH 300/339] glupload: use attached buffer-alignment when uploading a VideoFrame --- .../gst-plugins-base/gst-libs/gst/gl/gstglupload.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c index 0124ba89123..90efb24c05f 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c @@ -101,6 +101,8 @@ struct _GstGLUploadPrivate GstCaps *in_caps; GstCaps *out_caps; + GstVideoAlignment align; + GstBuffer *outbuf; /* all method impl pointers */ @@ -2076,7 +2078,9 @@ static struct RawUploadFrame * _raw_upload_frame_new (struct RawUpload *raw, GstBuffer * buffer) { struct RawUploadFrame *frame; + GstVideoFrame *vframe; GstVideoInfo *info; + GstVideoMeta *vmeta; gint i; if (!buffer) @@ -2084,13 +2088,17 @@ _raw_upload_frame_new (struct RawUpload *raw, GstBuffer * buffer) frame = g_new (struct RawUploadFrame, 1); frame->ref_count = 1; + vframe = &frame->frame; - if (!gst_video_frame_map (&frame->frame, &raw->upload->priv->in_info, + if (!gst_video_frame_map (vframe, &raw->upload->priv->in_info, buffer, GST_MAP_READ)) { g_free (frame); return NULL; } + vmeta = vframe->meta; + if (vmeta) + raw->upload->priv->align = vmeta->alignment; raw->upload->priv->in_info = frame->frame.info; info = &raw->upload->priv->in_info; @@ -2210,7 +2218,7 @@ _raw_data_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, gst_gl_allocation_params_free ((GstGLAllocationParams *) raw->params); if (!(raw->params = gst_gl_video_allocation_params_new_wrapped_data (raw->upload->context, - NULL, &raw->upload->priv->in_info, -1, NULL, + NULL, &raw->upload->priv->in_info, -1, &raw->upload->priv->align, GST_GL_TEXTURE_TARGET_2D, 0, NULL, raw->in_frame, (GDestroyNotify) _raw_upload_frame_unref))) return FALSE; @@ -3167,6 +3175,7 @@ static void gst_gl_upload_init (GstGLUpload * upload) { upload->priv = gst_gl_upload_get_instance_private (upload); + gst_video_alignment_reset (&upload->priv->align); } /** From 649dbf9ab6277f0a57ecd6575cf3b5742e051bdf Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 12 Apr 2023 15:55:32 +0200 Subject: [PATCH 301/339] amcvideoenc: default to -1 as I_FRAME_INTERVAL_DEFAULT This will produce a single I-frame at the start and then after that only on request. This makes much more sense than "0", which means producing an I-frame for every single frame! --- .../sys/androidmedia/gstamcvideoenc.c | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c index 7bb27a01fe4..ba0d7d341d1 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c @@ -94,7 +94,7 @@ static GstFlowReturn gst_amc_video_enc_finish (GstVideoEncoder * encoder); static GstFlowReturn gst_amc_video_enc_drain (GstAmcVideoEnc * self); #define BIT_RATE_DEFAULT (2 * 1024 * 1024) -#define I_FRAME_INTERVAL_DEFAULT 0 +#define I_FRAME_INTERVAL_DEFAULT -1 enum { PROP_0, @@ -576,7 +576,7 @@ gst_amc_video_enc_set_property (GObject * object, guint prop_id, break; case PROP_I_FRAME_INTERVAL: - encoder->i_frame_int = g_value_get_uint (value); + encoder->i_frame_int = g_value_get_int (value); if (codec_active) goto wrong_state; break; @@ -614,7 +614,7 @@ gst_amc_video_enc_get_property (GObject * object, guint prop_id, g_value_set_uint (value, encoder->bitrate); break; case PROP_I_FRAME_INTERVAL: - g_value_set_uint (value, encoder->i_frame_int); + g_value_set_int (value, encoder->i_frame_int); break; case PROP_I_FRAME_INTERVAL_FLOAT: g_value_set_float (value, encoder->i_frame_int); @@ -664,10 +664,19 @@ gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass) G_MAXINT, BIT_RATE_DEFAULT, dynamic_flag | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /* + From developer.android.com: + A key describing the frequency of key frames expressed in seconds + between key frames. + This key is used by video encoders. A negative value means no key frames + are requested after the first frame. + A zero value means a stream containing all key frames is requested. + */ g_object_class_install_property (gobject_class, PROP_I_FRAME_INTERVAL, - g_param_spec_uint ("i-frame-interval", "I-frame interval", - "The frequency of I frames expressed in seconds between I frames (0 for automatic)", - 0, G_MAXINT, I_FRAME_INTERVAL_DEFAULT, + g_param_spec_int ("i-frame-interval", "I-frame interval", + "The frequency of I frames expressed in seconds between I frames " + "(-1 for never after first, 0 for every frame)", + -1, G_MAXINT, I_FRAME_INTERVAL_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_I_FRAME_INTERVAL_FLOAT, From 829dca4f07b8a2119aa076adb8d82276a091d65c Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Tue, 18 Apr 2023 16:09:23 +0200 Subject: [PATCH 302/339] amcvideoenc: various fixes --- .../sys/androidmedia/gstamcvideoenc.c | 90 +++++++++++++------ 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c index ba0d7d341d1..781aa754191 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c @@ -266,7 +266,7 @@ create_amc_format (GstAmcVideoEnc * encoder, GstVideoCodecState * input_state, gst_amc_format_set_int (format, "color-format", color_format, &err); if (err) GST_ELEMENT_WARNING_FROM_ERROR (encoder, err); - stride = GST_ROUND_UP_4 (info->width); /* safe (?) */ + stride = info->width; gst_amc_format_set_int (format, "stride", stride, &err); if (err) GST_ELEMENT_WARNING_FROM_ERROR (encoder, err); @@ -626,6 +626,12 @@ gst_amc_video_enc_get_property (GObject * object, guint prop_id, GST_OBJECT_UNLOCK (encoder); } +static gboolean +gst_amc_video_enc_transform_meta (G_GNUC_UNUSED GstVideoEncoder * encoder, + G_GNUC_UNUSED GstVideoCodecFrame * frame, G_GNUC_UNUSED GstMeta * meta) +{ + return TRUE; +} static void gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass) @@ -653,6 +659,8 @@ gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass) videoenc_class->handle_frame = GST_DEBUG_FUNCPTR (gst_amc_video_enc_handle_frame); videoenc_class->finish = GST_DEBUG_FUNCPTR (gst_amc_video_enc_finish); + videoenc_class->transform_meta = + GST_DEBUG_FUNCPTR (gst_amc_video_enc_transform_meta); // On Android >= 19, we can set bitrate dynamically // so add the flag so apps can detect it. @@ -833,8 +841,8 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) GList *l, *best_l = NULL; GList *finish_frames = NULL; GstVideoCodecFrame *best = NULL; - guint64 best_timestamp = 0; - guint64 best_diff = G_MAXUINT64; + GstClockTime best_timestamp = 0; + GstClockTimeDiff best_diff = G_MAXINT64; BufferIdentification *best_id = NULL; GList *frames; @@ -843,21 +851,24 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) for (l = frames; l; l = l->next) { GstVideoCodecFrame *tmp = l->data; BufferIdentification *id = gst_video_codec_frame_get_user_data (tmp); - guint64 timestamp, diff; + GstClockTime timestamp; + GstClockTimeDiff diff; /* This happens for frames that were just added but * which were not passed to the component yet. Ignore * them here! */ - if (!id) + if (!id) { + GST_WARNING_OBJECT (self, "Found frame without BufferIdentification"); continue; + } timestamp = id->timestamp; - if (timestamp > reference_timestamp) - diff = timestamp - reference_timestamp; - else - diff = reference_timestamp - timestamp; + diff = ABS (GST_CLOCK_DIFF (timestamp, reference_timestamp)); + GST_LOG_OBJECT (self, + "Found frame with ID: %" GST_TIME_FORMAT " and diff: %" GST_TIME_FORMAT, + GST_TIME_ARGS (id->timestamp), GST_TIME_ARGS (diff)); if (best == NULL || diff < best_diff) { best = tmp; @@ -866,6 +877,8 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) best_l = l; best_id = id; + GST_DEBUG_OBJECT (self, "We got a best frame"); + /* For frames without timestamp we simply take the first frame */ if ((reference_timestamp == 0 && !GST_CLOCK_TIME_IS_VALID (timestamp)) || diff == 0) @@ -899,6 +912,8 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) if (finish_frames) { g_warning ("%s: Too old frames, bug in encoder -- please file a bug", GST_ELEMENT_NAME (self)); + GST_ERROR_OBJECT (self, + "Too old frames, bug in encoder -- please file a bug"); for (l = finish_frames; l; l = l->next) { gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (self), l->data); } @@ -920,6 +935,8 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) g_list_foreach (frames, (GFunc) gst_video_codec_frame_unref, NULL); g_list_free (frames); + GST_DEBUG_OBJECT (self, "Returning %p", best); + return best; } @@ -979,8 +996,11 @@ gst_amc_video_enc_fill_buffer (GstAmcVideoEnc * self, GstBuffer * inbuf, * then we can use state->info safely */ GstVideoInfo *info = &input_state->info; - if (buffer_info->size < self->color_format_info.frame_size) + if (buffer_info->size < self->color_format_info.frame_size) { + GST_WARNING_OBJECT (self, "Buffer size too small for frame (%u < %u)", + buffer_info->size, self->color_format_info.frame_size); return FALSE; + } return gst_amc_color_format_copy (&self->color_format_info, outbuf, buffer_info, info, inbuf, COLOR_FORMAT_COPY_IN); @@ -1014,6 +1034,10 @@ gst_amc_video_enc_handle_output_frame (GstAmcVideoEnc * self, if (frame) { frame->output_buffer = out_buf; + if (buffer_info->flags & BUFFER_FLAG_SYNC_FRAME) { + GST_DEBUG_OBJECT (self, "Frame is sync frame!"); + GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame); + } flow_ret = gst_video_encoder_finish_frame (encoder, frame); } else { /* This sometimes happens at EOS or if the input is not properly framed, @@ -1049,13 +1073,12 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) GST_VIDEO_ENCODER_STREAM_LOCK (self); retry: - GST_DEBUG_OBJECT (self, "Waiting for available output buffer"); + GST_LOG_OBJECT (self, "Waiting for available output buffer"); GST_VIDEO_ENCODER_STREAM_UNLOCK (self); - /* Wait at most 100ms here, some codecs don't fail dequeueing if + /* Wait at most 1s here, some codecs don't fail dequeueing if * the codec is flushing, causing deadlocks during shutdown */ - idx = - gst_amc_codec_dequeue_output_buffer (self->codec, &buffer_info, 100000, - &err); + idx = gst_amc_codec_dequeue_output_buffer (self->codec, &buffer_info, + G_USEC_PER_SEC, &err); GST_VIDEO_ENCODER_STREAM_LOCK (self); /*} */ @@ -1072,7 +1095,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) GstAmcFormat *format; gchar *format_string; - GST_DEBUG_OBJECT (self, "Output format has changed"); + GST_INFO_OBJECT (self, "Output format has changed"); format = (idx == INFO_OUTPUT_FORMAT_CHANGED) ? gst_amc_codec_get_output_format (self->codec, @@ -1096,7 +1119,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) gst_amc_format_free (format); goto format_error; } - GST_DEBUG_OBJECT (self, "Got new output format: %s", format_string); + GST_INFO_OBJECT (self, "Got new output format: %s", format_string); g_free (format_string); if (!gst_amc_video_enc_set_src_caps (self, format)) { @@ -1118,7 +1141,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) g_assert_not_reached (); break; case INFO_TRY_AGAIN_LATER: - GST_DEBUG_OBJECT (self, "Dequeueing output buffer timed out"); + GST_LOG_OBJECT (self, "Dequeueing output buffer timed out"); goto retry; break; case G_MININT: @@ -1134,7 +1157,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) } process_buffer: - GST_DEBUG_OBJECT (self, + GST_LOG_OBJECT (self, "Got output buffer at index %d: size %d time %" G_GINT64_FORMAT " flags 0x%08x", idx, buffer_info.size, buffer_info.presentation_time_us, buffer_info.flags); @@ -1232,7 +1255,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) g_mutex_unlock (&self->drain_lock); GST_VIDEO_ENCODER_STREAM_LOCK (self); } else { - GST_DEBUG_OBJECT (self, "Finished frame: %s", gst_flow_get_name (flow_ret)); + GST_LOG_OBJECT (self, "Finished frame: %s", gst_flow_get_name (flow_ret)); } self->downstream_flow_ret = flow_ret; @@ -1290,7 +1313,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) } flushing: { - GST_DEBUG_OBJECT (self, "Flushing -- stopping task"); + GST_WARNING_OBJECT (self, "Flushing -- stopping task"); gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self)); self->downstream_flow_ret = GST_FLOW_FLUSHING; GST_VIDEO_ENCODER_STREAM_UNLOCK (self); @@ -1300,7 +1323,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) flow_error: { if (flow_ret == GST_FLOW_EOS) { - GST_DEBUG_OBJECT (self, "EOS"); + GST_ERROR_OBJECT (self, "EOS"); gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ()); gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self)); @@ -1310,6 +1333,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) gst_event_new_eos ()); gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self)); } + self->downstream_flow_ret = flow_ret; GST_VIDEO_ENCODER_STREAM_UNLOCK (self); g_mutex_lock (&self->drain_lock); self->draining = FALSE; @@ -1574,7 +1598,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, self = GST_AMC_VIDEO_ENC (encoder); - GST_DEBUG_OBJECT (self, "Handling frame"); + GST_LOG_OBJECT (self, "Handling frame"); if (!self->started) { GST_ERROR_OBJECT (self, "Codec not started yet"); @@ -1593,7 +1617,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME (frame)) { if (gst_amc_codec_request_key_frame (self->codec, &err)) { - GST_DEBUG_OBJECT (self, "Passed keyframe request to MediaCodec"); + GST_INFO_OBJECT (self, "Passed keyframe request to MediaCodec"); } if (err) { GST_ELEMENT_WARNING_FROM_ERROR (self, err); @@ -1634,12 +1658,15 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, } if (self->flushing) { + GST_INFO_OBJECT (self, "We are flushing"); memset (&buffer_info, 0, sizeof (buffer_info)); gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, NULL); goto flushing; } if (self->downstream_flow_ret != GST_FLOW_OK) { + GST_INFO_OBJECT (self, "Downstream returned %s", + gst_flow_get_name (self->downstream_flow_ret)); memset (&buffer_info, 0, sizeof (buffer_info)); gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, &err); if (err && !self->flushing) @@ -1653,6 +1680,8 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, /* Copy the buffer content in chunks of size as requested * by the port */ buf = gst_amc_codec_get_input_buffer (self->codec, idx, &err); + GST_LOG_OBJECT (self, "Got input buffer: %" GST_PTR_FORMAT, buf); + if (err) goto failed_to_get_input_buffer; else if (!buf) @@ -1666,6 +1695,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, if (!gst_amc_video_enc_fill_buffer (self, frame->input_buffer, buf, &buffer_info)) { + GST_ERROR_OBJECT (self, "Could not fill buffer!"); memset (&buffer_info, 0, sizeof (buffer_info)); gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, &err); if (err && !self->flushing) @@ -1688,12 +1718,16 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, self->last_upstream_ts += duration; id = buffer_identification_new (timestamp + timestamp_offset); - if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) + GST_LOG_OBJECT (self, "Created buffer id with timestamp: %" + GST_TIME_FORMAT, GST_TIME_ARGS (id->timestamp)); + if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) { + GST_DEBUG_OBJECT (self, "Frame is sync point"); buffer_info.flags |= BUFFER_FLAG_SYNC_FRAME; + } gst_video_codec_frame_set_user_data (frame, id, (GDestroyNotify) buffer_identification_free); - GST_DEBUG_OBJECT (self, + GST_LOG_OBJECT (self, "Queueing buffer %d: size %d time %" G_GINT64_FORMAT " flags 0x%08x", idx, buffer_info.size, buffer_info.presentation_time_us, buffer_info.flags); @@ -1727,6 +1761,8 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, } got_null_input_buffer: { + GST_ERROR_OBJECT (self, "got_null_input_buffer"); + GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL), ("Got no input buffer")); gst_video_codec_frame_unref (frame); @@ -1734,6 +1770,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, } buffer_fill_error: { + GST_ERROR_OBJECT (self, "buffer_fill_error"); GST_ELEMENT_ERROR (self, RESOURCE, WRITE, (NULL), ("Failed to write input into the amc buffer(write %dB to a %" G_GSIZE_FORMAT "B buffer)", self->color_format_info.frame_size, @@ -1755,6 +1792,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, } flushing: { + GST_ERROR_OBJECT (self, "flushing"); GST_DEBUG_OBJECT (self, "Flushing -- returning FLUSHING"); gst_video_codec_frame_unref (frame); return GST_FLOW_FLUSHING; From 00239be756f88f6bf2f35c838a36e65b4fbdfc0b Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 2 Jun 2023 15:52:08 +0200 Subject: [PATCH 303/339] gstamc: various fixes --- .../sys/androidmedia/gstamc-constants.h | 2 +- .../sys/androidmedia/gstamcaudiodec.c | 2 +- .../sys/androidmedia/gstamcvideodec.c | 2 +- .../sys/androidmedia/gstamcvideoenc.c | 13 ++++++++++--- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc-constants.h b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc-constants.h index 95a965aed71..f7aaa6aa61d 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc-constants.h +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc-constants.h @@ -29,7 +29,7 @@ enum { - BUFFER_FLAG_SYNC_FRAME = 1, + BUFFER_FLAG_KEY_FRAME = 1, BUFFER_FLAG_CODEC_CONFIG = 2, BUFFER_FLAG_END_OF_STREAM = 4, BUFFER_FLAG_PARTIAL_FRAME = 8, diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcaudiodec.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcaudiodec.c index 398a6e1bd3b..4a88e3fc59f 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcaudiodec.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcaudiodec.c @@ -1179,7 +1179,7 @@ gst_amc_audio_dec_handle_frame (GstAudioDecoder * decoder, GstBuffer * inbuf) if (offset == 0) { if (!GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_DELTA_UNIT)) - buffer_info.flags |= BUFFER_FLAG_SYNC_FRAME; + buffer_info.flags |= BUFFER_FLAG_KEY_FRAME; } offset += buffer_info.size; diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideodec.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideodec.c index e9230fb3868..00b153a8089 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideodec.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideodec.c @@ -2212,7 +2212,7 @@ gst_amc_video_dec_handle_frame (GstVideoDecoder * decoder, BufferIdentification *id = buffer_identification_new (timestamp + timestamp_offset); if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) - buffer_info.flags |= BUFFER_FLAG_SYNC_FRAME; + buffer_info.flags |= BUFFER_FLAG_KEY_FRAME; gst_video_codec_frame_set_user_data (frame, id, (GDestroyNotify) buffer_identification_free); } diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c index 781aa754191..14eaad1d2a5 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c @@ -1031,13 +1031,20 @@ gst_amc_video_enc_handle_output_frame (GstAmcVideoEnc * self, GST_BUFFER_PTS (out_buf) = gst_util_uint64_scale (buffer_info->presentation_time_us, GST_USECOND, 1); + GST_BUFFER_DTS (out_buf) = GST_BUFFER_PTS (out_buf); + + if (!(buffer_info->flags & BUFFER_FLAG_PARTIAL_FRAME)) { + GST_DEBUG_OBJECT (self, "Frame is complete"); + GST_BUFFER_FLAG_SET (out_buf, GST_BUFFER_FLAG_MARKER); + } if (frame) { frame->output_buffer = out_buf; - if (buffer_info->flags & BUFFER_FLAG_SYNC_FRAME) { + if (buffer_info->flags & BUFFER_FLAG_KEY_FRAME) { GST_DEBUG_OBJECT (self, "Frame is sync frame!"); GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame); } + flow_ret = gst_video_encoder_finish_frame (encoder, frame); } else { /* This sometimes happens at EOS or if the input is not properly framed, @@ -1313,7 +1320,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) } flushing: { - GST_WARNING_OBJECT (self, "Flushing -- stopping task"); + GST_INFO_OBJECT (self, "Flushing -- stopping task"); gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self)); self->downstream_flow_ret = GST_FLOW_FLUSHING; GST_VIDEO_ENCODER_STREAM_UNLOCK (self); @@ -1722,7 +1729,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, GST_TIME_FORMAT, GST_TIME_ARGS (id->timestamp)); if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) { GST_DEBUG_OBJECT (self, "Frame is sync point"); - buffer_info.flags |= BUFFER_FLAG_SYNC_FRAME; + buffer_info.flags |= BUFFER_FLAG_KEY_FRAME; } gst_video_codec_frame_set_user_data (frame, id, (GDestroyNotify) buffer_identification_free); From 47f9d64a4f984f9f305049a15734c202cf26959f Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 24 Apr 2023 14:12:41 +0200 Subject: [PATCH 304/339] gstreamer-full-static: fix plugins option parse for Windows --- scripts/generate_init_static_plugins.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/generate_init_static_plugins.py b/scripts/generate_init_static_plugins.py index f490aafc69d..377c9fd0ef2 100755 --- a/scripts/generate_init_static_plugins.py +++ b/scripts/generate_init_static_plugins.py @@ -2,6 +2,7 @@ import argparse import os +import platform from string import Template TEMPLATE = Template(''' @@ -93,7 +94,7 @@ def process_features(features_list, plugins, feature_prefix): giomodules_declaration = [] giomodules_registration = [] - if ',' in options.plugins or ':' in options.plugins: + if ',' in options.plugins or (platform.system() != 'Windows' and ':' in options.plugins): print("Only ';' is allowed in the list of plugins.") exit(1) From 58c3aa9e0067080d1e0846a6883f5c9c06a91031 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 22 Feb 2023 13:25:12 +0100 Subject: [PATCH 305/339] netsim: introduce throttling The idea is basically that every n second, we delay each packet within a X ms window, so that they all burst out at the same time at the end of that window, recreating a throttling effect for a network, but without actually losing any packets. --- .../gst-plugins-bad/gst/netsim/gstnetsim.c | 74 ++++++++++++++++++- .../gst-plugins-bad/gst/netsim/gstnetsim.h | 3 + 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c index a6ac4e384d2..4b3afaa46b9 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c @@ -74,7 +74,9 @@ enum PROP_QUEUE_SIZE, PROP_MAX_QUEUE_DELAY, PROP_ALLOW_REORDERING, - PROP_REPLACE_DROPPED_WITH_EMPTY + PROP_REPLACE_DROPPED_WITH_EMPTY, + PROP_THROTTLE_FREQUENCY, + PROP_THROTTLE_DELAY, }; /* these numbers are nothing but wild guesses and don't reflect any reality */ @@ -91,6 +93,8 @@ enum #define DEFAULT_MAX_QUEUE_DELAY 50 #define DEFAULT_ALLOW_REORDERING TRUE #define DEFAULT_REPLACE_DROPPED_WITH_EMPTY FALSE +#define DEFAULT_THROTTLE_FREQUENCY 0 +#define DEFAULT_THROTTLE_DELAY 0 static GstStaticPadTemplate gst_net_sim_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -448,11 +452,53 @@ gst_new_sim_get_delay_ms (GstNetSim * netsim) return delay_ms; } +static gint +gst_new_sim_get_throttle_ms (GstNetSim * netsim, GstClockTime now) +{ + gint ret = 0; + GstClockTime start_time; + + /* we don't have a frequency or a delay, return 0 */ + if (!netsim->throttle_frequency || !netsim->throttle_delay) { + return 0; + } + + /* Invalid configuration */ + if (netsim->throttle_delay / 1000 > netsim->throttle_frequency) { + GST_ERROR_OBJECT (netsim, + "throttle-delay larger than frequency not allowed!"); + g_assert_not_reached (); + } + + /* we are not valid yet, set it to now */ + if (!GST_CLOCK_TIME_IS_VALID (netsim->throttle_end_time)) { + netsim->throttle_end_time = now; + } + + /* we are past our end-time, move it forward */ + if (now >= netsim->throttle_end_time) { + netsim->throttle_end_time += (netsim->throttle_frequency * GST_SECOND); + } + + /* when the throttling should start */ + start_time = + netsim->throttle_end_time - (netsim->throttle_delay * GST_MSECOND); + + /* we are within the throttling window, delay all packets until throttle_end_time */ + if (now >= start_time) { + ret = GST_TIME_AS_MSECONDS (netsim->throttle_end_time - now); + } + + return ret; +} + static void gst_net_sim_queue_buffer (GstNetSim * netsim, GstBuffer * buf) { GstClockTime now = gst_clock_get_time (netsim->clock); gint delay_ms = gst_new_sim_get_delay_ms (netsim); + delay_ms += gst_new_sim_get_throttle_ms (netsim, now); + NetSimBuffer *nsbuf = net_sim_buffer_new (buf, netsim->seqnum++, now, delay_ms * GST_MSECOND); @@ -665,6 +711,12 @@ gst_net_sim_set_property (GObject * object, case PROP_REPLACE_DROPPED_WITH_EMPTY: netsim->replace_droppped_with_empty = g_value_get_boolean (value); break; + case PROP_THROTTLE_FREQUENCY: + netsim->throttle_frequency = g_value_get_int (value); + break; + case PROP_THROTTLE_DELAY: + netsim->throttle_delay = g_value_get_int (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -717,6 +769,12 @@ gst_net_sim_get_property (GObject * object, case PROP_REPLACE_DROPPED_WITH_EMPTY: g_value_set_boolean (value, netsim->replace_droppped_with_empty); break; + case PROP_THROTTLE_FREQUENCY: + g_value_set_int (value, netsim->throttle_frequency); + break; + case PROP_THROTTLE_DELAY: + g_value_set_int (value, netsim->throttle_delay); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -740,6 +798,7 @@ gst_net_sim_init (GstNetSim * netsim) g_cond_init (&netsim->cond); netsim->rand_seed = g_rand_new (); netsim->prev_time = GST_CLOCK_TIME_NONE; + netsim->throttle_end_time = GST_CLOCK_TIME_NONE; GST_OBJECT_FLAG_SET (netsim->sinkpad, GST_PAD_FLAG_PROXY_CAPS | GST_PAD_FLAG_PROXY_ALLOCATION); @@ -920,6 +979,19 @@ gst_net_sim_class_init (GstNetSimClass * klass) DEFAULT_ALLOW_REORDERING, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_THROTTLE_FREQUENCY, + g_param_spec_int ("throttle-frequency", "Throttle Frequency (s)", + "How often (in seconds) a throttle should occur (0 = never)", + 0, G_MAXINT, DEFAULT_THROTTLE_FREQUENCY, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_THROTTLE_DELAY, + g_param_spec_int ("throttle-delay", "Throttle Delay (ms)", + "When a throttle occurs, for how long the packets should be held back", + 0, G_MAXINT, DEFAULT_THROTTLE_DELAY, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + GST_DEBUG_CATEGORY_INIT (netsim_debug, "netsim", 0, "Network simulator"); gst_type_mark_as_plugin_api (distribution_get_type (), 0); diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h index c5dcaa4cc54..fc0fa14608b 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h @@ -82,6 +82,7 @@ struct _GstNetSim guint seqnum; GCompareDataFunc compare_func; guint bits_in_queue; + GstClockTime throttle_end_time; /* properties */ gint min_delay; @@ -97,6 +98,8 @@ struct _GstNetSim gint max_queue_delay; gboolean allow_reordering; gboolean replace_droppped_with_empty; + gint throttle_frequency; + gint throttle_delay; }; struct _GstNetSimClass From bb2d5fa3b76c2d07488d7cb421df523df7c15e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Mon, 8 May 2023 12:53:31 +0200 Subject: [PATCH 306/339] netsim: improvements to the token bucket algorithm for limiting bw --- .../gst-plugins-bad/gst/netsim/gstnetsim.c | 220 ++++++++++-------- .../gst-plugins-bad/gst/netsim/gstnetsim.h | 2 +- 2 files changed, 124 insertions(+), 98 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c index 4b3afaa46b9..67db0a98c3b 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c @@ -116,10 +116,9 @@ typedef struct { GstBuffer *buf; guint size_bits; + guint seqnum; GstClockTime arrival_time; GstClockTime delay; - GstClockTime token_delay; - guint seqnum; } NetSimBuffer; static NetSimBuffer * @@ -132,14 +131,13 @@ net_sim_buffer_new (GstBuffer * buf, nsbuf->seqnum = seqnum; nsbuf->arrival_time = arrival_time; nsbuf->delay = delay; - nsbuf->token_delay = 0; return nsbuf; } static GstClockTime -net_sim_buffer_get_sync_time (NetSimBuffer * nsbuf) +net_sim_buffer_get_push_time (NetSimBuffer * nsbuf) { - return nsbuf->arrival_time + nsbuf->delay + nsbuf->token_delay; + return nsbuf->arrival_time + nsbuf->delay; } static void @@ -195,7 +193,8 @@ gst_net_sim_set_clock (GstElement * element, GstClock * clock) GST_NET_SIM_SIGNAL (netsim); GST_NET_SIM_UNLOCK (netsim); - return TRUE; + return GST_ELEMENT_CLASS (gst_net_sim_parent_class)->set_clock (element, + clock); } static gboolean @@ -298,18 +297,21 @@ get_random_value_gamma (GRand * rand_seed, gint32 low, gint32 high, return round (x + low); } -static gint -gst_net_sim_get_tokens (GstNetSim * netsim, GstClockTime now) +static void +gst_net_sim_add_tokens (GstNetSim * netsim, GstClockTime now) { gint tokens = 0; GstClockTimeDiff elapsed_time = 0; - GstClockTimeDiff token_time; - guint max_bps; + + if (netsim->max_kbps == -1 && netsim->max_bucket_size == -1) + return; /* check for umlimited kbps and fill up the bucket if that is the case, * if not, calculate the number of tokens to add based on the elapsed time */ - if (netsim->max_kbps == -1) - return netsim->max_bucket_size * 1000 - netsim->bucket_size; + if (netsim->max_kbps == -1) { + netsim->bucket_size = netsim->max_bucket_size * 8; + return; + } /* get the elapsed time */ if (GST_CLOCK_TIME_IS_VALID (netsim->prev_time)) { @@ -323,47 +325,59 @@ gst_net_sim_get_tokens (GstNetSim * netsim, GstClockTime now) } /* calculate number of tokens and how much time is "spent" by these tokens */ - max_bps = netsim->max_kbps * 1000; - tokens = gst_util_uint64_scale_int (elapsed_time, max_bps, GST_SECOND); - token_time = gst_util_uint64_scale_int (GST_SECOND, tokens, max_bps); + tokens = + gst_util_uint64_scale_int (elapsed_time, netsim->max_kbps * 1000, + GST_SECOND); GST_DEBUG_OBJECT (netsim, - "Elapsed time: %" GST_TIME_FORMAT " produces %u tokens (" "token-time: %" - GST_TIME_FORMAT ")", GST_TIME_ARGS (elapsed_time), tokens, - GST_TIME_ARGS (token_time)); + "Elapsed time: %" GST_TIME_FORMAT " produces %u tokens", + GST_TIME_ARGS (elapsed_time), tokens); - /* increment the time with how much we spent in terms of whole tokens */ - netsim->prev_time += token_time; - return tokens; + netsim->prev_time = now; + + netsim->bucket_size = + MIN (netsim->max_bucket_size * 8, netsim->bucket_size + tokens); + GST_LOG_OBJECT (netsim, "Added %d tokens to bucket (contains %u tokens)", + tokens, netsim->bucket_size); } -static guint -gst_net_sim_get_missing_tokens (GstNetSim * netsim, - NetSimBuffer * nsbuf, GstClockTime now) +static GstClockTime +gst_net_sim_get_missing_token_time (GstNetSim * netsim, NetSimBuffer * nsbuf) { - gint tokens; + GstClockTimeDiff ret; + gint missing_tokens; + gint tokens = nsbuf->size_bits; - /* with an unlimited bucket-size, we have nothing to do */ - if (netsim->max_bucket_size == -1) + /* unlimited bits per second and unlimited bucket size + means tokens are always available */ + if (netsim->max_kbps == -1 || netsim->max_bucket_size == -1) return 0; - tokens = gst_net_sim_get_tokens (netsim, now); + /* we we have room in our bucket, no need to wait */ + if (netsim->bucket_size >= tokens) + return 0; - netsim->bucket_size = - MIN (netsim->max_bucket_size * 1000, netsim->bucket_size + tokens); - GST_LOG_OBJECT (netsim, "Added %d tokens to bucket (contains %u tokens)", - tokens, netsim->bucket_size); + /* lets not divide by 0 */ + if (netsim->max_kbps == 0) + return 0; - if (nsbuf->size_bits > netsim->bucket_size) { - GST_DEBUG_OBJECT (netsim, "Buffer size (%u) exeedes bucket size (%u)", - nsbuf->size_bits, netsim->bucket_size); - return nsbuf->size_bits - netsim->bucket_size; - } + missing_tokens = tokens - netsim->bucket_size; + ret = + gst_util_uint64_scale (GST_SECOND, missing_tokens, + netsim->max_kbps * 1000); + + return ret; +} + +static void +gst_net_sim_take_tokens (GstNetSim * netsim, NetSimBuffer * nsbuf) +{ + if (netsim->max_kbps == -1 || netsim->max_bucket_size == -1) + return; netsim->bucket_size -= nsbuf->size_bits; - GST_DEBUG_OBJECT (netsim, "Buffer taking %u tokens (%u left)", + GST_DEBUG_OBJECT (netsim, "Buffer taking %u tokens (%d left)", nsbuf->size_bits, netsim->bucket_size); - return 0; } static void @@ -376,49 +390,71 @@ gst_net_sim_drop_nsbuf (GstNetSim * netsim) net_sim_buffer_free (nsbuf); } +static void +gst_net_sim_wait (GstNetSim * netsim, GstClockTime push_time) +{ + netsim->clock_id = gst_clock_new_single_shot_id (netsim->clock, push_time); + + GST_DEBUG_OBJECT (netsim, "waiting for push_time: %" GST_TIME_FORMAT, + GST_TIME_ARGS (push_time)); + + GST_NET_SIM_UNLOCK (netsim); + gst_clock_id_wait (netsim->clock_id, NULL); + GST_NET_SIM_LOCK (netsim); + + gst_clock_id_unref (netsim->clock_id); + netsim->clock_id = NULL; +} + /* must be called with GST_NET_SIM_LOCK */ static GstFlowReturn -gst_net_sim_push_unlocked (GstNetSim * netsim, GstClockTimeDiff jitter) +gst_net_sim_push_unlocked (GstNetSim * netsim) { GstFlowReturn ret = GST_FLOW_OK; GstClockTime now = gst_clock_get_time (netsim->clock); NetSimBuffer *nsbuf = g_queue_peek_head (netsim->bqueue); - GstClockTime sync_time = net_sim_buffer_get_sync_time (nsbuf); - guint missing_tokens; + GstClockTime push_time = net_sim_buffer_get_push_time (nsbuf); + GstClockTime token_delay; + + gst_net_sim_add_tokens (netsim, now); GST_DEBUG_OBJECT (netsim, - "now: %" GST_TIME_FORMAT " jitter %" GST_STIME_FORMAT - " netsim->max_delay %" GST_STIME_FORMAT, GST_TIME_ARGS (now), - GST_STIME_ARGS (jitter), - GST_STIME_ARGS (netsim->max_delay * GST_MSECOND)); - - missing_tokens = gst_net_sim_get_missing_tokens (netsim, nsbuf, now); - if (missing_tokens > 0) { - GstClockTime token_delay = gst_util_uint64_scale_int (GST_SECOND, - missing_tokens, netsim->max_kbps * 1000); - GstClockTime new_synctime = now + token_delay; - GstClockTime delta = new_synctime - net_sim_buffer_get_sync_time (nsbuf); - nsbuf->token_delay += delta; - - GST_DEBUG_OBJECT (netsim, - "Missing %u tokens, delaying buffer #%u additional %" GST_TIME_FORMAT - " (total: %" GST_TIME_FORMAT ") for new sync_time: %" GST_TIME_FORMAT, - missing_tokens, nsbuf->seqnum, GST_TIME_ARGS (delta), - GST_TIME_ARGS (nsbuf->token_delay), GST_TIME_ARGS (sync_time)); - - if (nsbuf->token_delay > netsim->max_queue_delay * GST_MSECOND) { + "now: %" GST_TIME_FORMAT " netsim->max_delay %" GST_STIME_FORMAT, + GST_TIME_ARGS (now), GST_STIME_ARGS (netsim->max_delay * GST_MSECOND)); + + token_delay = gst_net_sim_get_missing_token_time (netsim, nsbuf); + if (token_delay > 0) { + GstClockTime deadline = now + token_delay; + GstClockTimeDiff delay = GST_CLOCK_DIFF (push_time, deadline); + + /* if delay is too big, we drop */ + if (delay > netsim->max_queue_delay * GST_MSECOND) { + GST_DEBUG_OBJECT (netsim, + "Delay %ums > max_queue_delay %ums, dropping buffer", + (guint) GST_TIME_AS_MSECONDS (delay), netsim->max_queue_delay); gst_net_sim_drop_nsbuf (netsim); + return GST_FLOW_OK; + } else { + /* if not, we wait until the deadline and then push */ + GST_DEBUG_OBJECT (netsim, + "delaying buffer #%u an additional %" GST_TIME_FORMAT + "for new deadline: %" GST_TIME_FORMAT, + nsbuf->seqnum, GST_TIME_ARGS (token_delay), GST_TIME_ARGS (deadline)); + push_time += token_delay; } - } else { - GST_DEBUG_OBJECT (netsim, "Pushing buffer #%u now", nsbuf->seqnum); - nsbuf = g_queue_pop_head (netsim->bqueue); - netsim->bits_in_queue -= nsbuf->size_bits; - - GST_NET_SIM_UNLOCK (netsim); - ret = net_sim_buffer_push (nsbuf, netsim->srcpad); - GST_NET_SIM_LOCK (netsim); } + gst_net_sim_wait (netsim, push_time); + + GST_DEBUG_OBJECT (netsim, "Pushing buffer #%u now", nsbuf->seqnum); + nsbuf = g_queue_pop_head (netsim->bqueue); + gst_net_sim_take_tokens (netsim, nsbuf); + netsim->bits_in_queue -= nsbuf->size_bits; + + GST_NET_SIM_UNLOCK (netsim); + ret = net_sim_buffer_push (nsbuf, netsim->srcpad); + GST_NET_SIM_LOCK (netsim); + return ret; } @@ -495,7 +531,14 @@ gst_new_sim_get_throttle_ms (GstNetSim * netsim, GstClockTime now) static void gst_net_sim_queue_buffer (GstNetSim * netsim, GstBuffer * buf) { - GstClockTime now = gst_clock_get_time (netsim->clock); + GstClockTime now; + + if (!netsim->clock) { + GST_WARNING_OBJECT (netsim, "No clock, dropping buffer"); + return; + } + + now = gst_clock_get_time (netsim->clock); gint delay_ms = gst_new_sim_get_delay_ms (netsim); delay_ms += gst_new_sim_get_throttle_ms (netsim, now); @@ -506,7 +549,7 @@ gst_net_sim_queue_buffer (GstNetSim * netsim, GstBuffer * buf) GST_DEBUG_OBJECT (netsim, "Delaying buffer with %dms", delay_ms); } - GST_DEBUG_OBJECT (netsim, "queue_size: %u, bits_in_queue: %u, bufsize: %u", + GST_DEBUG_OBJECT (netsim, "queue_size: %d, bits_in_queue: %u, bufsize: %u", netsim->queue_size * 1000, netsim->bits_in_queue, nsbuf->size_bits); if (netsim->bits_in_queue > 0 && @@ -573,9 +616,6 @@ gst_net_sim_loop (gpointer data) { GstNetSim *netsim = GST_NET_SIM_CAST (data); GstFlowReturn ret; - NetSimBuffer *nsbuf; - GstClockTime sync_time; - GstClockTimeDiff jitter; GST_NET_SIM_LOCK (netsim); if (!gst_net_sim_wait_for_clock (netsim)) @@ -590,27 +630,13 @@ gst_net_sim_loop (gpointer data) if (!netsim->running) goto pause_task; - nsbuf = g_queue_peek_head (netsim->bqueue); - sync_time = net_sim_buffer_get_sync_time (nsbuf); - netsim->clock_id = gst_clock_new_single_shot_id (netsim->clock, sync_time); - - GST_DEBUG_OBJECT (netsim, "Popped buf #%u with sync_time: %" GST_TIME_FORMAT, - nsbuf->seqnum, GST_TIME_ARGS (sync_time)); - - GST_NET_SIM_UNLOCK (netsim); - gst_clock_id_wait (netsim->clock_id, &jitter); - GST_NET_SIM_LOCK (netsim); - - gst_clock_id_unref (netsim->clock_id); - netsim->clock_id = NULL; - - if (!netsim->running) { + ret = gst_net_sim_push_unlocked (netsim); + if (ret != GST_FLOW_OK) { + GST_ERROR_OBJECT (netsim, "pausing task because flow: %d", ret); goto pause_task; } - ret = gst_net_sim_push_unlocked (netsim, jitter); - if (ret != GST_FLOW_OK) { - GST_ERROR_OBJECT (netsim, "pausing task because flow: %d", ret); + if (!netsim->running) { goto pause_task; } @@ -693,7 +719,7 @@ gst_net_sim_set_property (GObject * object, case PROP_MAX_BUCKET_SIZE: netsim->max_bucket_size = g_value_get_int (value); if (netsim->max_bucket_size != -1) - netsim->bucket_size = netsim->max_bucket_size * 1000; + netsim->bucket_size = netsim->max_bucket_size * 8; break; case PROP_QUEUE_SIZE: netsim->queue_size = g_value_get_int (value); @@ -925,13 +951,13 @@ gst_net_sim_class_init (GstNetSimClass * klass) /** * GstNetSim:max-bucket-size: * - * The size of the token bucket, related to burstiness resilience. + * The size of the token bucket in Bytes. * * Since: 1.14 */ g_object_class_install_property (gobject_class, PROP_MAX_BUCKET_SIZE, - g_param_spec_int ("max-bucket-size", "Maximum Bucket Size (Kb)", - "The size of the token bucket, related to burstiness resilience " + g_param_spec_int ("max-bucket-size", "Maximum Bucket Size (B)", + "The size of the token bucket, in bytes " "(-1 = unlimited)", -1, G_MAXINT, DEFAULT_MAX_BUCKET_SIZE, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h index fc0fa14608b..066393e4844 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h @@ -69,7 +69,7 @@ struct _GstNetSim GstPad *srcpad; GRand *rand_seed; - guint bucket_size; + gint bucket_size; NormalDistributionState delay_state; GstClockTime prev_time; From b4a091dcfe73cd6e8c8d2f2271d79195625f26c3 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 4 May 2023 13:29:55 +0200 Subject: [PATCH 307/339] gst: use GST_API to properly import/export functions in gstevent and gsttask --- subprojects/gstreamer/gst/gstevent.h | 2 +- subprojects/gstreamer/gst/gsttask.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gstreamer/gst/gstevent.h b/subprojects/gstreamer/gst/gstevent.h index e7eb75a8604..7b1a1eef708 100644 --- a/subprojects/gstreamer/gst/gstevent.h +++ b/subprojects/gstreamer/gst/gstevent.h @@ -688,7 +688,7 @@ GST_API void gst_event_parse_latency (GstEvent *event, GstClockTime *latency); /* latency-changed event */ -GST_EXPORT +GST_API GstEvent* gst_event_new_latency_changed (void) G_GNUC_MALLOC; /* step event */ diff --git a/subprojects/gstreamer/gst/gsttask.h b/subprojects/gstreamer/gst/gsttask.h index 9a7415ad033..000f9ce70b0 100644 --- a/subprojects/gstreamer/gst/gsttask.h +++ b/subprojects/gstreamer/gst/gsttask.h @@ -214,7 +214,7 @@ gboolean gst_task_resume (GstTask *task); GST_API gboolean gst_task_join (GstTask *task); -GST_EXPORT +GST_API void gst_task_class_set_default_task_pool_type (GType type); G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstTask, gst_object_unref) From 329d0357d1c1e8eea884eae395a01745d96040cb Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Tue, 4 Jul 2023 22:46:56 +0900 Subject: [PATCH 308/339] rtpvp8pay: don't payload partitions when forced not to parse the frame regression from 63005d (cherry picked from commit eb702a5a0e0f45d4e64549a5831748d0f071f113) --- subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c index 26a83e990fb..1b8118ccbac 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c @@ -668,10 +668,12 @@ gst_rtp_vp8_payload_next (GstRtpVP8Pay * self, GstBufferList * list, if (available > remaining) available = remaining; - if (meta) { + if (meta || !self->parse_frames) { /* If meta is present, then we have no partition offset information, * so always emit PID 0 and set the start bit for the first packet - * of a frame only (c.f. RFC7741 $4.4) + * of a frame only (c.f. RFC7741 $4.4). Or, when we want to avoid parsing + * the bitstream as currently partition information is only present in + * the VP8 frame itself. */ partition = 0; start = (offset == 0); From 59a1d23b5f8989efed2853751f3d7e523b1c1e01 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 11 Jul 2023 18:37:55 +0200 Subject: [PATCH 309/339] audiodecoder: do not account for upstream latency when producing DTX packets - Removed the upstream latency as well, not needed anymore. --- .../gst-libs/gst/audio/gstaudiodecoder.c | 30 ------------------- .../gst-libs/gst/audio/gstaudiodecoder.h | 3 -- .../tests/check/libs/audiodecoder.c | 21 +++++-------- 3 files changed, 8 insertions(+), 46 deletions(-) diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c index b97bd2809d4..fd5f256dd84 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c @@ -264,9 +264,6 @@ struct _GstAudioDecoderPrivate /* context storage */ GstAudioDecoderContext ctx; - /* the cached upstream latency */ - GstClockTime us_latency; - /* properties */ GstClockTime latency; GstClockTime reported_min_latency; @@ -1723,10 +1720,8 @@ gst_audio_decoder_dtx_task_func (gpointer user_data) GstFlowReturn ret; GstBuffer *buf; - pts += dec->priv->us_latency; pts += dec->priv->last_buffer_duration; - gst_audio_decoder_dtx_wait_unlocked (dec, base_time + pts); if (!dec->priv->dtx_running) @@ -3212,8 +3207,6 @@ gst_audio_decoder_src_query_default (GstAudioDecoder * dec, GstQuery * query) else max_latency += dec->priv->ctx.max_latency; - /* store the upstream min latency */ - dec->priv->us_latency = min_latency; GST_OBJECT_UNLOCK (dec); gst_query_set_latency (query, live, min_latency, max_latency); @@ -3784,29 +3777,6 @@ gst_audio_decoder_get_min_latency (GstAudioDecoder * dec) return result; } -/** - * gst_audio_decoder_get_upstream_latency: - * @dec: a #GstAudioDecoder - * - * Returns: the upstream latency. - * - * MT safe. - */ -GstClockTime -gst_audio_decoder_get_upstream_latency (GstAudioDecoder * dec) -{ - GstClockTime result; - - g_return_val_if_fail (GST_IS_AUDIO_DECODER (dec), FALSE); - - GST_OBJECT_LOCK (dec); - result = dec->priv->us_latency; - GST_OBJECT_UNLOCK (dec); - - return result; -} - - /** * gst_audio_decoder_set_tolerance: * @dec: a #GstAudioDecoder diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h index 0653e45f231..d7fb323fd42 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h @@ -413,9 +413,6 @@ void gst_audio_decoder_set_min_latency (GstAudioDecoder * dec, GST_AUDIO_API GstClockTime gst_audio_decoder_get_min_latency (GstAudioDecoder * dec); -GST_AUDIO_API -GstClockTime gst_audio_decoder_get_upstream_latency (GstAudioDecoder * dec); - GST_AUDIO_API void gst_audio_decoder_set_tolerance (GstAudioDecoder * dec, GstClockTime tolerance); diff --git a/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c b/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c index c8fb165370c..96059c2732e 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c +++ b/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c @@ -303,13 +303,15 @@ fail_unless_silent_buffer (GstBuffer * buf) } static void -fail_unless_silent_buffers (GstHarness * h, gint count) +fail_unless_silent_buffers (GstHarness * h, gint count, GstClockTime pts) { for (gint i = 0; i < count; i++) { GstBuffer *buf; gst_harness_crank_single_clock_wait (h); buf = gst_harness_pull (h); + pts += GST_BUFFER_DURATION (buf); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); fail_unless_silent_buffer (buf); } } @@ -1214,9 +1216,10 @@ GST_START_TEST (audiodecoder_dtx) fail_unless_silent_buffer (buf); /* expect some silent buffers from the DTX thread */ - fail_unless_silent_buffers (h, 2); + fail_unless_silent_buffers (h, 2, pts); /* advance two packets */ pkt += 2; + pts += 2 * dur; /* and audio buffer arrives, the decoder should stop DTX */ buf = create_test_buffer (pkt++); @@ -1239,14 +1242,8 @@ GST_START_TEST (audiodecoder_dtx) GST_END_TEST; -static const int audiodecoder_lost_pkt_stops_dtx_latencies[] = { - 0, 10, 20, 50, 100, 300, 500, 600, -}; - GST_START_TEST (audiodecoder_lost_pkt_stops_dtx) { - gint latency_ms = audiodecoder_lost_pkt_stops_dtx_latencies[__i__]; - GstClockTime upstream_latency = latency_ms * GST_MSECOND; GstClockTime pts, dur; GstClockTime lost_pts, lost_dur; GstBuffer *buf; @@ -1254,7 +1251,6 @@ GST_START_TEST (audiodecoder_lost_pkt_stops_dtx) gst_audio_decoder_set_plc_aware (GST_AUDIO_DECODER (h->element), TRUE); gst_audio_decoder_set_plc (GST_AUDIO_DECODER (h->element), TRUE); - gst_harness_set_upstream_latency (h, upstream_latency); /* start with DTX in */ buf = create_dtx_buffer (0); @@ -1268,8 +1264,8 @@ GST_START_TEST (audiodecoder_lost_pkt_stops_dtx) fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless_silent_buffer (buf); - /* lets receive 3 silent buffers */ - fail_unless_silent_buffers (h, 3); + /* lets receive 5 silent buffers */ + fail_unless_silent_buffers (h, 5, pts); /* buffer 4 is lost, dtx should be stopped now */ buf = create_test_buffer (4); @@ -1372,8 +1368,7 @@ gst_audiodecoder_suite (void) suite_add_tcase (s, tc = tcase_create ("dtx")); tcase_add_test (tc, audiodecoder_dtx); - tcase_add_loop_test (tc, audiodecoder_lost_pkt_stops_dtx, 0, - G_N_ELEMENTS (audiodecoder_lost_pkt_stops_dtx_latencies)); + tcase_add_test (tc, audiodecoder_lost_pkt_stops_dtx); tcase_add_test (tc, audiodecoder_dtx_stress_push); return s; From a112f471cbdeb11b1dd42dd9d848ad2417380dba Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 12 Jul 2023 11:31:32 +0200 Subject: [PATCH 310/339] gstahcsrc: override GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE property --- subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c index 448e11827d2..67349b6a821 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c @@ -447,12 +447,10 @@ gst_ahc_src_class_init (GstAHCSrcClass * klass) properties[PROP_APERTURE] = g_object_class_find_property (gobject_class, GST_PHOTOGRAPHY_PROP_APERTURE); -#if 0 g_object_class_override_property (gobject_class, PROP_EXPOSURE_MODE, GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE); - properties[PROP_EXPOSURE] = g_object_class_find_property (gobject_class, + properties[PROP_EXPOSURE_MODE] = g_object_class_find_property (gobject_class, GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE); -#endif g_object_class_override_property (gobject_class, PROP_IMAGE_CAPTURE_SUPPORTED_CAPS, From 8bb0ac2b52dc48b2f759cf89fc39e9fce246565e Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 12 Jul 2023 11:40:10 +0200 Subject: [PATCH 311/339] gstamcvideoenc: fixed minimum value for PROP_I_FRAME_INTERVAL_FLOAT Fixes g_param_spec_float: assertion 'default_value >= minimum && default_value <= maximum' failed --- subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c index 14eaad1d2a5..5d7f9e68a11 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c @@ -691,7 +691,7 @@ gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass) g_param_spec_float ("i-frame-interval-float", "I-frame interval", "The frequency of I frames expressed in seconds between I frames (0 for automatic). " "Fractional intervals work on Android >= 25", - 0, G_MAXFLOAT, I_FRAME_INTERVAL_DEFAULT, + -1, G_MAXFLOAT, I_FRAME_INTERVAL_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } From da9d8f730df22859d1769951a728459b49e7c1fe Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 29 Aug 2023 15:26:48 +0200 Subject: [PATCH 312/339] applemedia/vtenc: reset the encoder session if we fail to encode a frame - kVTInvalidSessionErr: common if the application goes between fg and bg mode. - kVTVideoEncoderMalfunctionErr: if the encoder has an internal error. --- .../gst-plugins-bad/sys/applemedia/vtenc.c | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c index ebfd6fc10b4..63badf29a51 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c @@ -1032,6 +1032,13 @@ gst_vtenc_is_negotiated (GstVTEnc * self) return self->session && self->video_info.width != 0; } +static void +gst_vtenc_reset_session (GstVTEnc * self) +{ + gst_vtenc_destroy_session (self, &self->session); + self->session = gst_vtenc_create_session (self); +} + /* * When the image is opaque but the output ProRes format has an alpha * component (4 component, 32 bits per pixel), Apple requires that we signal @@ -1943,14 +1950,19 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) GINT_TO_POINTER (frame->system_frame_number), NULL); GST_VIDEO_ENCODER_STREAM_LOCK (self); - if (vt_status != noErr) { - GST_WARNING_OBJECT (self, "VTCompressionSessionEncodeFrame returned %d", - (int) vt_status); - } - gst_video_codec_frame_unref (frame); CVPixelBufferRelease (pbuf); + if (vt_status == kVTInvalidSessionErr) { + GST_WARNING_OBJECT (self, "Invalid compression session, resetting."); + gst_vtenc_reset_session (self); + } else if (vt_status == kVTVideoEncoderMalfunctionErr) { + GST_WARNING_OBJECT (self, "Invalid compression session, resetting."); + gst_vtenc_reset_session (self); + } else if (vt_status != noErr) { + GST_WARNING_OBJECT (self, "VTCompressionSessionEncodeFrame returned %d", (int) vt_status); + } + return ret; drop: From 463649e5076a3b35bfc1080e84ef9fab32b58cbf Mon Sep 17 00:00:00 2001 From: Camilo Celis Guzman Date: Wed, 30 Aug 2023 18:40:55 +0900 Subject: [PATCH 313/339] rtpvp9pay: extract resolution from sink caps (#9) --- .../gst-plugins-good/gst/rtp/gstrtpvp9pay.c | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c index f17cc91a71b..8aeb88d114e 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c @@ -634,6 +634,30 @@ gst_rtp_vp9_pay_handle_buffer (GstRTPBasePayload * payload, GstBuffer * buffer) return ret; } +static void +gst_rtp_vp9_pay_set_resolution_from_event (GstRtpVP9Pay * self, + const GstEvent * event) +{ + GstCaps * caps = NULL; + GstStructure * s = NULL; + + gst_event_parse_caps (event, &caps); + s = gst_caps_get_structure (caps, 0); + + gint width = 0, height = 0; + if (gst_structure_get_int (s, "width", &width) + && gst_structure_get_int (s, "height", &height) + && width > 0 && height > 0) { + GST_INFO_OBJECT (self, "Parsed resolution from caps: %dx%d -> %dx%d", + self->width, self->height, width, height); + self->width = width; + self->height = height; + } else { + GST_INFO_OBJECT (self, "Failed to parse resolution from caps: " + "%dx%d -/-> %dx%d",self->width, self->height, width, height); + } +} + static gboolean gst_rtp_vp9_pay_sink_event (GstRTPBasePayload * payload, GstEvent * event) { @@ -641,10 +665,12 @@ gst_rtp_vp9_pay_sink_event (GstRTPBasePayload * payload, GstEvent * event) GstEventType event_type = GST_EVENT_TYPE (event); if (event_type == GST_EVENT_GAP || event_type == GST_EVENT_FLUSH_START) { - gint picture_id = self->picture_id; - gst_rtp_vp9_pay_picture_id_increment (self); - GST_DEBUG_OBJECT (payload, "Incrementing picture ID on %s event %d -> %d", - GST_EVENT_TYPE_NAME (event), picture_id, self->picture_id); + gint picture_id = self->picture_id; + gst_rtp_vp9_pay_picture_id_increment (self); + GST_DEBUG_OBJECT (payload, "Incrementing picture ID on %s event %d -> %d", + GST_EVENT_TYPE_NAME (event), picture_id, self->picture_id); + } else if (event_type == GST_EVENT_CAPS && !self->parse_frames) { + gst_rtp_vp9_pay_set_resolution_from_event (self, event); } return GST_RTP_BASE_PAYLOAD_CLASS (gst_rtp_vp9_pay_parent_class)->sink_event From 14fe9e10eb38cc1ffd440a17c2c53853f3f89211 Mon Sep 17 00:00:00 2001 From: Vadym Markov Date: Thu, 31 Aug 2023 12:27:14 +0200 Subject: [PATCH 314/339] vulkan: fix building for simulator --- .../gst-plugins-bad/gst-libs/gst/vulkan/meson.build | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build index d8b0069a3b6..e92206669f0 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build @@ -88,6 +88,7 @@ has_vulkan_header = false vulkan_dep = dependency('', required: false) vulkan_inc_dir = '' enabled_vulkan_winsys = [] +is_ios_simulator = host_system == 'ios' and meson.get_external_property('needs_exe_wrapper') vulkan_conf = configuration_data() vulkan_conf_options = [ @@ -120,7 +121,15 @@ if ['ios', 'darwin'].contains(host_system) if vulkan_root != '' and vulkan_root != 'None' molten_vk_root = join_paths(vulkan_root, 'MoltenVK') - platform_dir = host_system == 'ios' ? 'ios-arm64' : 'macos-arm64_x86_64' + + if is_ios_simulator + platform_dir = 'ios-arm64_x86_64-simulator' + elif host_system == 'ios' + platform_dir = 'ios-arm64' + else + platform_dir = 'macos-arm64_x86_64' + endif + vulkan_lib_dir = join_paths(molten_vk_root, 'MoltenVK.xcframework', platform_dir) vulkan_inc_dir = join_paths(molten_vk_root, 'include') vulkan_lib = cc.find_library('MoltenVK', dirs: vulkan_lib_dir, From bbbd63373718704e9f92b72d93d164f8930684c6 Mon Sep 17 00:00:00 2001 From: Vadym Markov Date: Thu, 31 Aug 2023 13:58:26 +0200 Subject: [PATCH 315/339] vulkan: set default value for needs_exe_wrapper external property --- subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build index e92206669f0..0a6524f8731 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build @@ -88,7 +88,7 @@ has_vulkan_header = false vulkan_dep = dependency('', required: false) vulkan_inc_dir = '' enabled_vulkan_winsys = [] -is_ios_simulator = host_system == 'ios' and meson.get_external_property('needs_exe_wrapper') +is_ios_simulator = host_system == 'ios' and meson.get_external_property('needs_exe_wrapper', false) vulkan_conf = configuration_data() vulkan_conf_options = [ From 363ed96b1b430de51981dd7d9a52a00953f0e1b3 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Tue, 5 Sep 2023 16:12:37 +0200 Subject: [PATCH 316/339] vulkan: implemented set_window_handle in gstvkwindow_cocoa --- .../gst/vulkan/cocoa/gstvkwindow_cocoa.m | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m index 8e4a91e1e2f..1ffa6ed2e42 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m @@ -62,6 +62,7 @@ { gpointer internal_win_id; gpointer internal_view; + gpointer external_view; gint preferred_width; gint preferred_height; @@ -80,6 +81,8 @@ static gboolean gst_vulkan_window_cocoa_get_presentation_support (GstVulkanWindo static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, GError ** error); static void gst_vulkan_window_cocoa_close (GstVulkanWindow * window); +static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, + guintptr window_handle); static void gst_vulkan_window_cocoa_finalize (GObject * object) @@ -100,6 +103,7 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, window_class->get_surface = gst_vulkan_window_cocoa_get_surface; window_class->get_presentation_support = gst_vulkan_window_cocoa_get_presentation_support; + window_class->set_window_handle = gst_vulkan_window_cocoa_set_window_handle; } static void @@ -154,9 +158,14 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (window); GstVulkanWindowCocoaPrivate *priv = GET_PRIV (window_cocoa); - if (!priv->visible) - _gst_vk_invoke_on_main ((GstVulkanWindowFunc) _show_window, - gst_object_ref (window), (GDestroyNotify) gst_object_unref); + if (!priv->visible) { + if (priv->external_view) { + gst_vulkan_window_cocoa_set_window_handle (window, (guintptr) priv->external_view); + } else { + _gst_vk_invoke_on_main ((GstVulkanWindowFunc) _show_window, + gst_object_ref (window), (GDestroyNotify) gst_object_unref); + } + } } static void @@ -191,6 +200,10 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, priv->internal_win_id = (__bridge_retained gpointer)internal_win_id; priv->internal_view = (__bridge_retained gpointer)view; + if (priv->external_view) + gst_vulkan_window_cocoa_set_window_handle (GST_VULKAN_WINDOW (window_cocoa), + (guintptr) priv->external_view); + gst_vulkan_window_cocoa_show (GST_VULKAN_WINDOW (window_cocoa)); } @@ -245,6 +258,49 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, return TRUE; } +static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, + guintptr handle) +{ + GST_ERROR("set_window_handle !!!! window_handle : %p", handle); + + GstVulkanWindowCocoa *window_cocoa; + GstVulkanWindowCocoaPrivate *priv; + + window_cocoa = GST_VULKAN_WINDOW_COCOA (window); + priv = GET_PRIV(window_cocoa); + + if (priv->internal_win_id) { + if (handle) { + priv->external_view = (gpointer)handle; + priv->visible = TRUE; + } else { + /* bring back our internal window */ + priv->external_view = 0; + priv->visible = FALSE; + } + + dispatch_async (dispatch_get_main_queue (), ^{ + GstVulkanNSWindow *internal_win_id = + (__bridge GstVulkanNSWindow *)priv->internal_win_id; + NSView *external_view = + (__bridge NSView *)priv->external_view; + NSView *view = (__bridge NSView *)priv->internal_view; + + [internal_win_id orderOut:internal_win_id]; + + [external_view addSubview: view]; + + [external_view setAutoresizesSubviews: YES]; + [view setFrame: [external_view bounds]]; + [view setAutoresizingMask: NSViewWidthSizable|NSViewHeightSizable]; + }); + } else { + /* no internal window yet so delay it to the next drawing */ + priv->external_view = (gpointer)handle; + priv->visible = FALSE; + } +} + static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, GError ** error) { From cbb4bb98f5f28f28a377dd53b39864733dedb0d0 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 6 Sep 2023 09:31:24 +0200 Subject: [PATCH 317/339] vulkan: removed extra log --- .../gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m | 2 -- 1 file changed, 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m index 1ffa6ed2e42..525875009c7 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m @@ -261,8 +261,6 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, guintptr handle) { - GST_ERROR("set_window_handle !!!! window_handle : %p", handle); - GstVulkanWindowCocoa *window_cocoa; GstVulkanWindowCocoaPrivate *priv; From 241072e09718e4e7268829d1298a45d22b988c1c Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Wed, 6 Sep 2023 15:08:12 +0200 Subject: [PATCH 318/339] vulkan: gstvkwindow_cocoa use _gst_vulkan_cocoa_invoke_on_main rather than dispatch_async on set_window_handle --- .../gst/vulkan/cocoa/gstvkwindow_cocoa.m | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m index 525875009c7..9617db1ec2a 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m @@ -258,7 +258,29 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, return TRUE; } -static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, +static void +_gst_vulkan_window_cocoa_insert_internal_win (gpointer data) +{ + GstVulkanWindowCocoa *window_cocoa; + GstVulkanWindowCocoaPrivate *priv; + + window_cocoa = (GstVulkanWindowCocoa*) data; + priv = GET_PRIV(window_cocoa); + + GstVulkanNSWindow *internal_win_id = (__bridge GstVulkanNSWindow *)priv->internal_win_id; + NSView *external_view = (__bridge NSView *)priv->external_view; + NSView *view = (__bridge NSView *)priv->internal_view; + + [internal_win_id orderOut:internal_win_id]; + [external_view addSubview: view]; + + [external_view setAutoresizesSubviews: YES]; + [view setFrame: [external_view bounds]]; + [view setAutoresizingMask: NSViewWidthSizable|NSViewHeightSizable]; +} + +static void +gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, guintptr handle) { GstVulkanWindowCocoa *window_cocoa; @@ -277,21 +299,9 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, priv->visible = FALSE; } - dispatch_async (dispatch_get_main_queue (), ^{ - GstVulkanNSWindow *internal_win_id = - (__bridge GstVulkanNSWindow *)priv->internal_win_id; - NSView *external_view = - (__bridge NSView *)priv->external_view; - NSView *view = (__bridge NSView *)priv->internal_view; - - [internal_win_id orderOut:internal_win_id]; - - [external_view addSubview: view]; + _gst_vulkan_cocoa_invoke_on_main ((GstVulkanWindowFunc) _gst_vulkan_window_cocoa_insert_internal_win, + gst_object_ref (window), (GDestroyNotify) gst_object_unref); - [external_view setAutoresizesSubviews: YES]; - [view setFrame: [external_view bounds]]; - [view setAutoresizingMask: NSViewWidthSizable|NSViewHeightSizable]; - }); } else { /* no internal window yet so delay it to the next drawing */ priv->external_view = (gpointer)handle; From a9e447dcf6a5ec6dafe6ebf630e1a7f2a6b815a6 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Thu, 7 Sep 2023 09:50:33 +0200 Subject: [PATCH 319/339] vulkan: fixed main thread checker warning CreateMacOSSurface should only be called from the main thread. --- .../gst/vulkan/cocoa/gstvkwindow_cocoa.m | 78 +++++++++++++++++-- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m index 9617db1ec2a..69ad8e50980 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m @@ -218,20 +218,84 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, return TRUE; } -static VkSurfaceKHR -gst_vulkan_window_cocoa_get_surface (GstVulkanWindow * window, GError ** error) +typedef struct +{ + VkResult result; + VkSurfaceKHR surface; + VkMacOSSurfaceCreateInfoMVK * info; + GstVulkanWindow * window; + + GMutex mutex; + GCond cond; + gboolean called; +} GstVulkanCococa_CreateMacOSSurfaceCtx; + +static void +_gst_vulkan_cocoa_wait_for_createmacossurface (GstVulkanCococa_CreateMacOSSurfaceCtx * ctx) +{ + g_mutex_lock (&ctx->mutex); + while (!ctx->called) + g_cond_wait(&ctx->cond, &ctx->mutex); + g_mutex_unlock (&ctx->mutex); + + g_mutex_clear (&ctx->mutex); + g_cond_clear (&ctx->cond); +} + +static void +_gst_vulkan_cocoa_create_macos_surface_on_main_thread (gpointer data) { + GstVulkanCococa_CreateMacOSSurfaceCtx * ctx = data; + GstVulkanWindow *window = ctx->window; GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (window); - GstVulkanWindowCocoaPrivate *priv = GET_PRIV (window_cocoa); - VkMacOSSurfaceCreateInfoMVK info = { 0, }; - VkSurfaceKHR ret; + + ctx->result = + window_cocoa->CreateMacOSSurface (window->display->instance->instance, ctx->info, + NULL, &ctx->surface); + + g_mutex_lock (&ctx->mutex); + ctx->called = TRUE; + g_cond_signal (&ctx->cond); + g_mutex_unlock (&ctx->mutex); +} + +static VkResult +_gst_vulkan_cocoa_create_macos_surface (GstVulkanWindow *window, VkSurfaceKHR * surface) +{ VkResult err; + VkMacOSSurfaceCreateInfoMVK info = { 0, }; + + GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (window); + GstVulkanWindowCocoaPrivate *priv = GET_PRIV (window_cocoa); + + GstVulkanCococa_CreateMacOSSurfaceCtx ctx; info.sType = VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK; info.pNext = NULL; info.flags = 0; info.pView = priv->internal_view; + memset(&ctx, 0, sizeof(GstVulkanCococa_CreateMacOSSurfaceCtx)); + g_mutex_init (&ctx.mutex); + g_cond_init (&ctx.cond); + ctx.info = &info; + ctx.window = window; + + _gst_vulkan_cocoa_invoke_on_main ((GstVulkanWindowFunc) _gst_vulkan_cocoa_create_macos_surface_on_main_thread, &ctx, NULL); + _gst_vulkan_cocoa_wait_for_createmacossurface (&ctx); + + *surface = ctx.surface; + + return ctx.result; +} + +static VkSurfaceKHR +gst_vulkan_window_cocoa_get_surface (GstVulkanWindow * window, GError ** error) +{ + GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (window); + VkSurfaceKHR ret; + VkResult err; + if (!window_cocoa->CreateMacOSSurface) window_cocoa->CreateMacOSSurface = gst_vulkan_instance_get_proc_address (window->display->instance, @@ -242,9 +306,7 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, return VK_NULL_HANDLE; } - err = - window_cocoa->CreateMacOSSurface (window->display->instance->instance, &info, - NULL, &ret); + err = _gst_vulkan_cocoa_create_macos_surface (window, &ret); if (gst_vulkan_error_to_g_error (err, error, "vkCreateMacOSSurfaceMVK") < 0) return VK_NULL_HANDLE; From e383c8bce48f07dca3e090d0916e28800f4dcb37 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 11 Sep 2023 09:48:03 +0200 Subject: [PATCH 320/339] gstaes: fixed missing symbols and rework on dependencies In our dynamic build winsock2 symbols are missing. --- subprojects/gst-plugins-bad/ext/aes/meson.build | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/aes/meson.build b/subprojects/gst-plugins-bad/ext/aes/meson.build index 169fb83c49d..abdeeb33796 100644 --- a/subprojects/gst-plugins-bad/ext/aes/meson.build +++ b/subprojects/gst-plugins-bad/ext/aes/meson.build @@ -6,20 +6,22 @@ aes_sources = [ ] aes_cargs = [] -aes_dep = dependency('openssl', version : '>= 1.1.0', required : get_option('aes')) -if aes_dep.found() +openssl_dep = dependency('openssl', version : '>= 1.1.0', required : get_option('aes')) +if openssl_dep.found() aes_cargs += ['-DHAVE_OPENSSL'] else subdir_done() endif +winsock2_dep = cc.find_library('ws2_32', required : false) +gst_aes_deps = [gstpbutils_dep, gstvideo_dep, openssl_dep, winsock2_dep, gio_dep, libm] + gstaes = library('gstaes', aes_sources, c_args : gst_plugins_bad_args + aes_cargs, link_args : noseh_link_args, include_directories : [configinc], - dependencies : [gstpbutils_dep, gstvideo_dep, - aes_dep, gio_dep, libm], + dependencies : gst_aes_deps, install : true, install_dir : plugins_install_dir, ) From ac947d3e09817072f63c0aa35135f570ea2a1e2c Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 11 Sep 2023 16:01:03 +0200 Subject: [PATCH 321/339] gstvkwindow: do not depend on close signal handlers to close the window gstvkwindow_cocoa: call close on the internal window Fixes window leak. --- .../gst/vulkan/cocoa/gstvkwindow_cocoa.m | 7 +++++-- .../gst-libs/gst/vulkan/gstvkwindow.c | 20 +++---------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m index 69ad8e50980..ed431e80767 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m @@ -386,7 +386,7 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, _close_window (gpointer * data) { GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (data); - GstVulkanWindow *window = GST_VULKAN_WINDOW (window_cocoa); + GstVulkanWindow *window = GST_VULKAN_WINDOW (window_cocoa) ; GstVulkanWindowCocoaPrivate *priv = GET_PRIV (window_cocoa); GstVulkanNSWindow *internal_win_id = (__bridge GstVulkanNSWindow *) priv->internal_win_id; @@ -394,9 +394,12 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, gst_vulkan_window_cocoa_hide (window); [[internal_win_id contentView] removeFromSuperview]; + [internal_win_id close]; + CFBridgingRelease (priv->internal_win_id); - priv->internal_win_id = NULL; CFBridgingRelease (priv->internal_view); + + priv->internal_win_id = NULL; priv->internal_view = NULL; } diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkwindow.c b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkwindow.c index da2834a2b00..3993b874f40 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkwindow.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkwindow.c @@ -106,18 +106,6 @@ enum static guint gst_vulkan_window_signals[LAST_SIGNAL] = { 0 }; -static gboolean -_accum_logical_and (GSignalInvocationHint * ihint, GValue * return_accu, - const GValue * handler_return, gpointer data) -{ - gboolean val = g_value_get_boolean (handler_return); - gboolean val2 = g_value_get_boolean (return_accu); - - g_value_set_boolean (return_accu, val && val2); - - return TRUE; -} - GQuark gst_vulkan_window_error_quark (void) { @@ -194,7 +182,7 @@ gst_vulkan_window_class_init (GstVulkanWindowClass * klass) gst_vulkan_window_signals[SIGNAL_CLOSE] = g_signal_new ("close", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST, 0, - (GSignalAccumulator) _accum_logical_and, NULL, NULL, G_TYPE_BOOLEAN, 0); + NULL, NULL, NULL, G_TYPE_NONE, 0); gst_vulkan_window_signals[SIGNAL_DRAW] = g_signal_new ("draw", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST, 0, @@ -410,16 +398,14 @@ void gst_vulkan_window_close (GstVulkanWindow * window) { GstVulkanWindowClass *klass; - gboolean to_close; g_return_if_fail (GST_IS_VULKAN_WINDOW (window)); klass = GST_VULKAN_WINDOW_GET_CLASS (window); g_return_if_fail (klass->close != NULL); - g_signal_emit (window, gst_vulkan_window_signals[SIGNAL_CLOSE], 0, &to_close); + g_signal_emit (window, gst_vulkan_window_signals[SIGNAL_CLOSE], 0); - if (to_close) - klass->close (window); + klass->close (window); } /** From 556f447b4db26c9ce0ea3d5ac728ac194538afe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Sun, 1 Oct 2023 16:11:52 +0100 Subject: [PATCH 322/339] PEXHACK: remove 'medias' from gst-integration-testsuites --- .gitmodules | 3 --- subprojects/gst-integration-testsuites/medias | 1 - 2 files changed, 4 deletions(-) delete mode 160000 subprojects/gst-integration-testsuites/medias diff --git a/.gitmodules b/.gitmodules index 9a950a1ed72..e69de29bb2d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "subprojects/gst-integration-testsuites/medias"] - path = subprojects/gst-integration-testsuites/medias - url = https://gitlab.freedesktop.org/gstreamer/gst-integration-testsuites.git diff --git a/subprojects/gst-integration-testsuites/medias b/subprojects/gst-integration-testsuites/medias deleted file mode 160000 index 121f8909493..00000000000 --- a/subprojects/gst-integration-testsuites/medias +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 121f8909493df214564c3db7fbba1dcb9217348e From 4d56783726302da316864e624d778dd2bc9fdd10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Mon, 2 Oct 2023 00:30:10 +0100 Subject: [PATCH 323/339] VPX: FIXUP: ISA BASED BUILD SYSTEM FOR LIBVPX --- subprojects/gst-plugins-good/ext/vpx/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/meson.build b/subprojects/gst-plugins-good/ext/vpx/meson.build index 096204f5285..0355018e2e1 100644 --- a/subprojects/gst-plugins-good/ext/vpx/meson.build +++ b/subprojects/gst-plugins-good/ext/vpx/meson.build @@ -123,7 +123,7 @@ foreach isa_flags : pex_isas vpxisa_dep = dependency('vpx-@0@'.format(isa), required : false) if vpxisa_dep.found() - gstvpxisa = shared_library('gstvpx_' + isa, + gstvpxisa = library('gstvpx_' + isa, vpx_sources, gstvpx_enums, c_args : gst_plugins_good_args + vpx_args + flags, include_directories: [configinc], @@ -131,5 +131,6 @@ foreach isa_flags : pex_isas install : true, install_dir : plugins_install_dir_isa + isa, ) + plugins += [gstvpxisa] endif endforeach From 393023dcf7dfb4fa13ec69bb0a343b20058c17a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Mon, 2 Oct 2023 09:59:10 +0100 Subject: [PATCH 324/339] FIXUP: VPX: we can't do vpx isa as a static library for now --- subprojects/gst-plugins-good/ext/vpx/meson.build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/ext/vpx/meson.build b/subprojects/gst-plugins-good/ext/vpx/meson.build index 0355018e2e1..8acff35530e 100644 --- a/subprojects/gst-plugins-good/ext/vpx/meson.build +++ b/subprojects/gst-plugins-good/ext/vpx/meson.build @@ -95,7 +95,9 @@ if vpx_dep.found() endif - +if get_option('default_library') == 'static' + subdir_done() +endif #Pexip spesific From edd59707317c95ab3efdcd45277b02d204267480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Sat, 14 Oct 2023 16:12:43 +0200 Subject: [PATCH 325/339] avvidenc: start PTS from 0 --- subprojects/gst-libav/ext/libav/gstavvidenc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-libav/ext/libav/gstavvidenc.c b/subprojects/gst-libav/ext/libav/gstavvidenc.c index fef5cae9041..6aaebdcca63 100644 --- a/subprojects/gst-libav/ext/libav/gstavvidenc.c +++ b/subprojects/gst-libav/ext/libav/gstavvidenc.c @@ -932,7 +932,7 @@ gst_ffmpegvidenc_start (GstVideoEncoder * encoder) return FALSE; } - gst_video_encoder_set_min_pts (encoder, GST_SECOND * 60 * 60 * 1000); + gst_video_encoder_set_min_pts (encoder, 0); return TRUE; } From 51e46df68db0aab5d600a63e00c5b48facb1bc14 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Fri, 13 Oct 2023 16:05:42 +0200 Subject: [PATCH 326/339] avviddec: request sync point on decoding errors And introduce "require-keyfram" property --- subprojects/gst-libav/ext/libav/gstavviddec.c | 39 +++++++++++++++++-- subprojects/gst-libav/ext/libav/gstavviddec.h | 3 ++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-libav/ext/libav/gstavviddec.c b/subprojects/gst-libav/ext/libav/gstavviddec.c index 3763f56c6b5..03b90607c9d 100644 --- a/subprojects/gst-libav/ext/libav/gstavviddec.c +++ b/subprojects/gst-libav/ext/libav/gstavviddec.c @@ -48,6 +48,7 @@ GST_DEBUG_CATEGORY_STATIC (GST_CAT_PERFORMANCE); #define DEFAULT_DIRECT_RENDERING TRUE #define DEFAULT_MAX_THREADS 0 #define DEFAULT_OUTPUT_CORRUPT TRUE +#define DEFAULT_REQUIRE_KEYFRAME FALSE #define REQUIRED_POOL_MAX_BUFFERS 32 #define DEFAULT_STRIDE_ALIGN 31 #define DEFAULT_ALLOC_PARAM { 0, DEFAULT_STRIDE_ALIGN, 0, 0, } @@ -65,6 +66,7 @@ enum PROP_OUTPUT_CORRUPT, PROP_THREAD_TYPE, PROP_STD_COMPLIANCE, + PROP_REQUIRE_KEYFRAME, PROP_LAST }; @@ -295,6 +297,11 @@ gst_ffmpegviddec_subclass_init (GstFFMpegVidDecClass * klass, g_param_spec_boolean ("output-corrupt", "Output corrupt buffers", "Whether libav should output frames even if corrupted", DEFAULT_OUTPUT_CORRUPT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_REQUIRE_KEYFRAME, + g_param_spec_boolean ("require-keyframe", "Require keyframe", + "Whether the first frame is required to be a keyframe", + DEFAULT_REQUIRE_KEYFRAME, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); caps = klass->in_plugin->capabilities; if (caps & (AV_CODEC_CAP_FRAME_THREADS | AV_CODEC_CAP_SLICE_THREADS)) { @@ -351,12 +358,14 @@ gst_ffmpegviddec_subinit (GstFFMpegVidDec * ffmpegdec) ffmpegdec->output_corrupt = DEFAULT_OUTPUT_CORRUPT; ffmpegdec->thread_type = DEFAULT_THREAD_TYPE; ffmpegdec->std_compliance = DEFAULT_STD_COMPLIANCE; + ffmpegdec->require_keyframe = DEFAULT_REQUIRE_KEYFRAME; GST_PAD_SET_ACCEPT_TEMPLATE (GST_VIDEO_DECODER_SINK_PAD (ffmpegdec)); gst_video_decoder_set_use_default_pad_acceptcaps (GST_VIDEO_DECODER_CAST (ffmpegdec), TRUE); gst_video_decoder_set_needs_format (GST_VIDEO_DECODER (ffmpegdec), TRUE); + gst_video_decoder_set_needs_sync_point (GST_VIDEO_DECODER (ffmpegdec), TRUE); } static void @@ -1792,6 +1801,8 @@ gst_ffmpegviddec_video_frame (GstFFMpegVidDec * ffmpegdec, } else if (res < 0) { GST_VIDEO_DECODER_ERROR (ffmpegdec, 1, STREAM, DECODE, (NULL), ("Video decoding error"), *ret); + gst_video_decoder_request_sync_point (GST_VIDEO_DECODER_CAST (ffmpegdec), + frame, GST_VIDEO_DECODER_REQUEST_SYNC_POINT_DISCARD_INPUT); goto beach; } @@ -1942,8 +1953,6 @@ gst_ffmpegviddec_video_frame (GstFFMpegVidDec * ffmpegdec, GST_TIME_FORMAT, tmp, tmp->system_frame_number, GST_TIME_ARGS (tmp->pts), GST_TIME_ARGS (tmp->dts)); /* drop extra ref and remove from frame list */ - GST_VIDEO_CODEC_FRAME_FLAG_UNSET (tmp, - GST_FFMPEG_VIDEO_CODEC_FRAME_FLAG_ALLOCATED); gst_video_decoder_release_frame (dec, tmp); } else { /* drop extra ref we got */ @@ -1954,15 +1963,31 @@ gst_ffmpegviddec_video_frame (GstFFMpegVidDec * ffmpegdec, g_list_free (ol); } - av_frame_unref (ffmpegdec->picture); + if (ffmpegdec->picture->key_frame || + ffmpegdec->picture->pict_type == AV_PICTURE_TYPE_I) + GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (out_frame); + else + GST_VIDEO_CODEC_FRAME_UNSET_SYNC_POINT (out_frame); if (frame) GST_VIDEO_CODEC_FRAME_FLAG_UNSET (frame, GST_FFMPEG_VIDEO_CODEC_FRAME_FLAG_ALLOCATED); + GST_LOG_OBJECT (ffmpegdec, "out_frame: pict_type %d, key_frame %d -> sync %d", + ffmpegdec->picture->pict_type, ffmpegdec->picture->key_frame, + GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (out_frame)); if (gst_video_decoder_get_subframe_mode (GST_VIDEO_DECODER (ffmpegdec))) gst_video_decoder_have_last_subframe (GST_VIDEO_DECODER (ffmpegdec), out_frame); + av_frame_unref (ffmpegdec->picture); + + if (ffmpegdec->requiring_keyframe) { + if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (out_frame)) + ffmpegdec->requiring_keyframe = FALSE; + else + gst_video_decoder_request_sync_point (GST_VIDEO_DECODER_CAST (ffmpegdec), + frame, GST_VIDEO_DECODER_REQUEST_SYNC_POINT_DISCARD_INPUT); + } /* FIXME: Ideally we would remap the buffer read-only now before pushing but * libav might still have a reference to it! @@ -2158,6 +2183,8 @@ gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder, */ GST_VIDEO_DECODER_STREAM_UNLOCK (ffmpegdec); if (avcodec_send_packet (ffmpegdec->context, &packet) < 0) { + gst_video_decoder_request_sync_point (GST_VIDEO_DECODER_CAST (ffmpegdec), + frame, GST_VIDEO_DECODER_REQUEST_SYNC_POINT_DISCARD_INPUT); GST_VIDEO_DECODER_STREAM_LOCK (ffmpegdec); av_packet_free_side_data (&packet); goto send_packet_failed; @@ -2209,6 +2236,8 @@ gst_ffmpegviddec_start (GstVideoDecoder * decoder) ffmpegdec->context->opaque = ffmpegdec; GST_OBJECT_UNLOCK (ffmpegdec); + ffmpegdec->requiring_keyframe = ffmpegdec->require_keyframe; + return TRUE; } @@ -2486,6 +2515,8 @@ gst_ffmpegviddec_set_property (GObject * object, break; case PROP_STD_COMPLIANCE: ffmpegdec->std_compliance = g_value_get_enum (value); + case PROP_REQUIRE_KEYFRAME: + ffmpegdec->require_keyframe = g_value_get_boolean (value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -2525,6 +2556,8 @@ gst_ffmpegviddec_get_property (GObject * object, break; case PROP_STD_COMPLIANCE: g_value_set_enum (value, ffmpegdec->std_compliance); + case PROP_REQUIRE_KEYFRAME: + g_value_set_boolean (value, ffmpegdec->require_keyframe); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); diff --git a/subprojects/gst-libav/ext/libav/gstavviddec.h b/subprojects/gst-libav/ext/libav/gstavviddec.h index 0f713de5691..068aa203e42 100644 --- a/subprojects/gst-libav/ext/libav/gstavviddec.h +++ b/subprojects/gst-libav/ext/libav/gstavviddec.h @@ -94,6 +94,9 @@ struct _GstFFMpegVidDec gint pool_height; enum AVPixelFormat pool_format; GstVideoInfo pool_info; + + gboolean requiring_keyframe; + gboolean require_keyframe; }; typedef struct _GstFFMpegVidDecClass GstFFMpegVidDecClass; From 3af68bbd7afc270bbd7f37a248559010d4b879d7 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 30 Oct 2023 09:16:47 +0100 Subject: [PATCH 327/339] gstvkwindow: fixup, use _gst_vk_invoke_on_main --- .../gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m index ed431e80767..6c6237dc50b 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m @@ -281,7 +281,7 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, ctx.info = &info; ctx.window = window; - _gst_vulkan_cocoa_invoke_on_main ((GstVulkanWindowFunc) _gst_vulkan_cocoa_create_macos_surface_on_main_thread, &ctx, NULL); + _gst_vk_invoke_on_main ((GstVulkanWindowFunc) _gst_vulkan_cocoa_create_macos_surface_on_main_thread, &ctx, NULL); _gst_vulkan_cocoa_wait_for_createmacossurface (&ctx); *surface = ctx.surface; @@ -361,7 +361,7 @@ static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, priv->visible = FALSE; } - _gst_vulkan_cocoa_invoke_on_main ((GstVulkanWindowFunc) _gst_vulkan_window_cocoa_insert_internal_win, + _gst_vk_invoke_on_main ((GstVulkanWindowFunc) _gst_vulkan_window_cocoa_insert_internal_win, gst_object_ref (window), (GDestroyNotify) gst_object_unref); } else { From cb59c9843bf791c987c883a385be3075d0b585cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Wed, 15 Nov 2023 10:43:01 +0100 Subject: [PATCH 328/339] applemedia/vtenc: fix deadlock on shutdown When shutting down the element, there is a fairly big chance that the loop is currently waiting for more buffers. At the point when (de)activate pad is called as part of the state changes, it will then deadlock because it want to aquire the stream-lock, and that is held by the loop... The fix (and this is the proper fix for ALL element that uses a GstTask) is to implement activatemode_function on the srcpad, so that we have a chance to stop/pause the loop when the pad is deactivated, to avoid such deadlocks. --- .../gst-plugins-bad/sys/applemedia/vtenc.c | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c index 63badf29a51..f8a1cdf0f9f 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c @@ -148,7 +148,8 @@ static void gst_vtenc_get_property (GObject * obj, guint prop_id, static void gst_vtenc_set_property (GObject * obj, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_vtenc_finalize (GObject * obj); - +static gboolean gst_vtenc_src_activate_mode (G_GNUC_UNUSED GstPad * pad, + GstObject * parent, GstPadMode mode, gboolean active); static gboolean gst_vtenc_start (GstVideoEncoder * enc); static gboolean gst_vtenc_stop (GstVideoEncoder * enc); static void gst_vtenc_loop (GstVTEnc * self); @@ -422,6 +423,7 @@ gst_vtenc_class_init (GstVTEncClass * klass) static void gst_vtenc_init (GstVTEnc * self) { + GstVideoEncoder *enc = GST_VIDEO_ENCODER_CAST (self); GstVTEncClass *klass = (GstVTEncClass *) G_OBJECT_GET_CLASS (self); CFStringRef keyframe_props_keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame }; CFBooleanRef keyframe_props_values[] = { kCFBooleanTrue }; @@ -443,6 +445,10 @@ gst_vtenc_init (GstVTEnc * self) g_mutex_init (&self->queue_mutex); g_cond_init (&self->queue_cond); + + gst_pad_set_activatemode_function (enc->srcpad, + GST_DEBUG_FUNCPTR (gst_vtenc_src_activate_mode)); + } static void @@ -698,6 +704,19 @@ gst_vtenc_pause_output_loop (GstVTEnc * self) g_mutex_unlock (&self->queue_mutex); } +static gboolean +gst_vtenc_src_activate_mode (G_GNUC_UNUSED GstPad * pad, + GstObject * parent, GstPadMode mode, gboolean active) +{ + GstVTEnc *self = GST_VTENC_CAST (parent); + GST_LOG_OBJECT (self, "activate mode %d active %d", mode, active); + + if (active == FALSE) + gst_vtenc_pause_output_loop (self); + + return TRUE; +} + static void gst_vtenc_set_flushing_flag (GstVTEnc * self) { @@ -830,6 +849,8 @@ gst_vtenc_stop (GstVideoEncoder * enc) return TRUE; } + + static gboolean gst_vtenc_h264_parse_profile_level_key (GstVTEnc * self, const gchar * profile, const gchar * level_arg) From 72758041d307dcdf0cc8dec3db9acbd20ad7d522 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 27 Nov 2023 09:57:47 +0100 Subject: [PATCH 329/339] avviddec: fixup on missing break after PROP_STD_COMPLIANCE --- subprojects/gst-libav/ext/libav/gstavviddec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/subprojects/gst-libav/ext/libav/gstavviddec.c b/subprojects/gst-libav/ext/libav/gstavviddec.c index 03b90607c9d..d67cf4a4cf0 100644 --- a/subprojects/gst-libav/ext/libav/gstavviddec.c +++ b/subprojects/gst-libav/ext/libav/gstavviddec.c @@ -2556,6 +2556,7 @@ gst_ffmpegviddec_get_property (GObject * object, break; case PROP_STD_COMPLIANCE: g_value_set_enum (value, ffmpegdec->std_compliance); + break; case PROP_REQUIRE_KEYFRAME: g_value_set_boolean (value, ffmpegdec->require_keyframe); break; From 8156c71b78bf0a5f5ea3d6b3c756a3f635cb8dd4 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 29 Nov 2023 15:04:43 +0100 Subject: [PATCH 330/339] basetransform: cache in- and outcaps Instead of calling gst_pad_get_current_caps() twice for every buffer, we simply cache the caps and pass that instead. --- .../libs/gst/base/gstbasetransform.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/base/gstbasetransform.c b/subprojects/gstreamer/libs/gst/base/gstbasetransform.c index f4998cc1f32..52b1ed95ff0 100644 --- a/subprojects/gstreamer/libs/gst/base/gstbasetransform.c +++ b/subprojects/gstreamer/libs/gst/base/gstbasetransform.c @@ -180,6 +180,9 @@ struct _GstBaseTransformPrivate gsize cache_caps2_size; gboolean have_same_caps; + GstCaps *cache_incaps; + GstCaps *cache_outcaps; + gboolean negotiated; /* QoS *//* with LOCK */ @@ -1029,6 +1032,9 @@ gst_base_transform_configure_caps (GstBaseTransform * trans, GstCaps * in, gst_caps_replace (&priv->cache_caps1, NULL); gst_caps_replace (&priv->cache_caps2, NULL); + gst_caps_replace (&priv->cache_incaps, in); + gst_caps_replace (&priv->cache_outcaps, out); + /* figure out same caps state */ priv->have_same_caps = gst_caps_is_equal (in, out); GST_DEBUG_OBJECT (trans, "have_same_caps: %d", priv->have_same_caps); @@ -1708,8 +1714,8 @@ default_prepare_output_buffer (GstBaseTransform * trans, } /* else use the transform function to get the size */ - incaps = gst_pad_get_current_caps (trans->sinkpad); - outcaps = gst_pad_get_current_caps (trans->srcpad); + incaps = priv->cache_incaps; + outcaps = priv->cache_outcaps; /* srcpad might be flushing already if we're being shut down */ if (outcaps == NULL) @@ -1721,9 +1727,6 @@ default_prepare_output_buffer (GstBaseTransform * trans, res = gst_base_transform_transform_size (trans, GST_PAD_SINK, incaps, insize, outcaps, &outsize); - gst_caps_unref (incaps); - gst_caps_unref (outcaps); - if (!res) goto unknown_size; @@ -2078,8 +2081,7 @@ default_submit_input_buffer (GstBaseTransform * trans, gboolean is_discont, * or if the class doesn't implement a set_caps function (in which case it doesn't * care about caps) */ - if (!priv->negotiated && !priv->passthrough && - (bclass->set_caps != NULL)) + if (!priv->negotiated && !priv->passthrough && (bclass->set_caps != NULL)) goto not_negotiated; GST_OBJECT_LOCK (trans); @@ -2532,6 +2534,9 @@ gst_base_transform_activate (GstBaseTransform * trans, gboolean active) gst_caps_replace (&priv->cache_caps1, NULL); gst_caps_replace (&priv->cache_caps2, NULL); + gst_caps_replace (&priv->cache_incaps, NULL); + gst_caps_replace (&priv->cache_outcaps, NULL); + /* Make sure any left over buffer is freed */ gst_buffer_replace (&trans->queued_buf, NULL); From 2c8f6d96fffb25e646dc3012e5dc77df6f3edcf8 Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Fri, 1 Dec 2023 10:35:29 +0100 Subject: [PATCH 331/339] v4l2src: attempt to set the current camera format after open is called To test if the device is actually being hold by another process or not. --- .../gst-plugins-good/sys/v4l2/gstv4l2src.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2src.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2src.c index 07f8718794b..f711e1ac94c 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2src.c +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2src.c @@ -1128,6 +1128,21 @@ gst_v4l2src_change_state (GstElement * element, GstStateChange transition) gst_v4l2_error (v4l2src, &error); return GST_STATE_CHANGE_FAILURE; } + + /* the device might still busy even after open() succeds, so try also to + set the format, it will return an EBUSY error if the device is + already acquired by another process */ + GstCaps *caps = gst_v4l2_object_get_caps (obj, NULL); + caps = gst_caps_make_writable (caps); + gboolean set_format = gst_v4l2src_set_format (v4l2src, caps, &error); + gst_caps_unref (caps); + + if (!set_format) { + gst_v4l2_error (v4l2src, &error); + gst_v4l2_object_close (obj); + return GST_STATE_CHANGE_FAILURE; + } + break; default: break; From 5032eec98c34d6916dff522fa0581e26271b1083 Mon Sep 17 00:00:00 2001 From: Havard Graff Date: Wed, 9 Mar 2022 16:28:11 +0100 Subject: [PATCH 332/339] gstreamer/libs/base: initial commit of GstBaseIdleSrc A new src that spawns a single thread when it needs to push something, and then joins said thread and goes back to an idle state, not consuming system resources. Co-authored-by: Camilo Celis Guzman --- subprojects/gstreamer/libs/gst/base/base.h | 1 + .../gstreamer/libs/gst/base/gstbaseidlesrc.c | 1688 +++++++++++++++++ .../gstreamer/libs/gst/base/gstbaseidlesrc.h | 220 +++ .../gstreamer/libs/gst/base/meson.build | 3 + .../gstreamer/tests/check/libs/baseidlesrc.c | 112 ++ .../gstreamer/tests/check/libs/basesrc.c | 2 +- subprojects/gstreamer/tests/check/meson.build | 1 + 7 files changed, 2026 insertions(+), 1 deletion(-) create mode 100644 subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c create mode 100644 subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h create mode 100644 subprojects/gstreamer/tests/check/libs/baseidlesrc.c diff --git a/subprojects/gstreamer/libs/gst/base/base.h b/subprojects/gstreamer/libs/gst/base/base.h index a802a21feca..fcb99feb425 100644 --- a/subprojects/gstreamer/libs/gst/base/base.h +++ b/subprojects/gstreamer/libs/gst/base/base.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include diff --git a/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c new file mode 100644 index 00000000000..394c6f39253 --- /dev/null +++ b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c @@ -0,0 +1,1688 @@ +/* GStreamer + * Copyright (C) 1999,2000 Erik Walthinsen + * 2000,2005 Wim Taymans + * 2023 Havard Graff + * 2023 Camilo Celis Guzman + * + * gstbaseidlesrc.c: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstbaseidlesrc + * @title: GstBaseIdleSrc + * @short_description: Base class for getrange based source elements + * @see_also: #GstPushSrc, #GstBaseTransform, #GstBaseSink + * + + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include + +#include +#include + +#include "gstbaseidlesrc.h" + +GST_DEBUG_CATEGORY_STATIC (gst_base_idle_src_debug); +#define GST_CAT_DEFAULT gst_base_idle_src_debug + +/* BaseIdleSrc signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +#define DEFAULT_DO_TIMESTAMP FALSE + +enum +{ + PROP_0, + PROP_DO_TIMESTAMP +}; + +/* The src implementation need to respect the following locking order: + * 1. STREAM_LOCK + * 2. LIVE_LOCK + * 3. OBJECT_LOCK + */ +struct _GstBaseIdleSrcPrivate +{ + /* if a stream-start event should be sent */ + gboolean stream_start_pending; /* STREAM_LOCK */ + + /* if segment should be sent and a + * seqnum if it was originated by a seek */ + gboolean segment_pending; /* OBJECT_LOCK */ + guint32 segment_seqnum; /* OBJECT_LOCK */ + + /* startup latency is the time it takes between going to PLAYING and producing + * the first BUFFER with running_time 0. This value is included in the latency + * reporting. */ + GstClockTime latency; /* OBJECT_LOCK */ + /* timestamp offset, this is the offset add to the values of gst_times for + * pseudo live sources */ + GstClockTimeDiff ts_offset; /* OBJECT_LOCK */ + + gboolean do_timestamp; /* OBJECT_LOCK */ + + /* QoS *//* with LOCK */ + gdouble proportion; /* OBJECT_LOCK */ + GstClockTime earliest_time; /* OBJECT_LOCK */ + + GstBufferPool *pool; /* OBJECT_LOCK */ + GstAllocator *allocator; /* OBJECT_LOCK */ + GstAllocationParams params; /* OBJECT_LOCK */ + + GQueue *obj_queue; + GThread *thread; +}; + +static GstElementClass *parent_class = NULL; +static gint private_offset = 0; + + +static void gst_base_idle_src_start_task (GstBaseIdleSrc * src, gboolean wait); +static void gst_base_idle_src_process_object_queue (GstBaseIdleSrc * src); + +static inline GstBaseIdleSrcPrivate * +gst_base_idle_src_get_instance_private (GstBaseIdleSrc * self) +{ + return (G_STRUCT_MEMBER_P (self, private_offset)); +} + +/* TODO: do we support anything other than _BUFFER/_BYTES ? */ +/** + * gst_base_idle_src_set_format: + * @src: base source instance + * @format: the format to use + * + * Sets the default format of the source. This will be the format used + * for sending SEGMENT events and for performing seeks. + * + * If a format of GST_FORMAT_BYTES is set, the element will be able to + * operate in pull mode if the #GstBaseIdleSrcClass::is_seekable returns %TRUE. + * + * This function must only be called in states < %GST_STATE_PAUSED. + */ +void +gst_base_idle_src_set_format (GstBaseIdleSrc * src, GstFormat format) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + g_return_if_fail (GST_STATE (src) <= GST_STATE_READY); + + GST_OBJECT_LOCK (src); + gst_segment_init (&src->segment, format); + GST_OBJECT_UNLOCK (src); +} + +/** + * gst_base_idle_src_query_latency: + * @src: the source + * @live: (out) (allow-none): if the source is live + * @min_latency: (out) (allow-none): the min latency of the source + * @max_latency: (out) (allow-none): the max latency of the source + * + * Query the source for the latency parameters. @live will be %TRUE when @src is + * configured as a live source. @min_latency and @max_latency will be set + * to the difference between the running time and the timestamp of the first + * buffer. + * + * This function is mostly used by subclasses. + * + * Returns: %TRUE if the query succeeded. + */ +gboolean +gst_base_idle_src_query_latency (GstBaseIdleSrc * src, gboolean * live, + GstClockTime * min_latency, GstClockTime * max_latency) +{ + GstClockTime min; + + g_return_val_if_fail (GST_IS_BASE_IDLE_SRC (src), FALSE); + + GST_OBJECT_LOCK (src); + if (live) + *live = src->is_live; + + /* if we have a startup latency, report this one, else report 0. Subclasses + * are supposed to override the query function if they want something + * else. */ + if (src->priv->latency != -1) + min = src->priv->latency; + else + min = 0; + + if (min_latency) + *min_latency = min; + if (max_latency) + *max_latency = min; + + GST_LOG_OBJECT (src, "latency: live %d, min %" GST_TIME_FORMAT + ", max %" GST_TIME_FORMAT, src->is_live, GST_TIME_ARGS (min), + GST_TIME_ARGS (min)); + GST_OBJECT_UNLOCK (src); + + return TRUE; +} + +/** + * gst_base_idle_src_set_do_timestamp: + * @src: the source + * @timestamp: enable or disable timestamping + * + * Configure @src to automatically timestamp outgoing buffers based on the + * current running_time of the pipeline. This property is mostly useful for live + * sources. + */ +void +gst_base_idle_src_set_do_timestamp (GstBaseIdleSrc * src, gboolean timestamp) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + + GST_OBJECT_LOCK (src); + src->priv->do_timestamp = timestamp; + if (timestamp && src->segment.format != GST_FORMAT_TIME) + gst_segment_init (&src->segment, GST_FORMAT_TIME); + GST_OBJECT_UNLOCK (src); +} + +/** + * gst_base_idle_src_get_do_timestamp: + * @src: the source + * + * Query if @src timestamps outgoing buffers based on the current running_time. + * + * Returns: %TRUE if the base class will automatically timestamp outgoing buffers. + */ +gboolean +gst_base_idle_src_get_do_timestamp (GstBaseIdleSrc * src) +{ + gboolean res; + + g_return_val_if_fail (GST_IS_BASE_IDLE_SRC (src), FALSE); + + GST_OBJECT_LOCK (src); + res = src->priv->do_timestamp; + GST_OBJECT_UNLOCK (src); + + return res; +} + +/** + * gst_base_idle_src_new_segment: + * @src: a #GstBaseIdleSrc + * @segment: a pointer to a #GstSegment + * + * Prepare a new segment for emission downstream. This function must + * only be called by derived sub-classes, and only from the #GstBaseIdleSrcClass::create function, + * as the stream-lock needs to be held. + * + * The format for the @segment must be identical with the current format + * of the source, as configured with gst_base_idle_src_set_format(). + * + * The format of @src must not be %GST_FORMAT_UNDEFINED and the format + * should be configured via gst_base_idle_src_set_format() before calling this method. + * + * Returns: %TRUE if preparation of new segment succeeded. + * + * Since: 1.18 + */ +gboolean +gst_base_idle_src_new_segment (GstBaseIdleSrc * src, const GstSegment * segment) +{ + g_return_val_if_fail (GST_IS_BASE_IDLE_SRC (src), FALSE); + g_return_val_if_fail (segment != NULL, FALSE); + + GST_OBJECT_LOCK (src); + + if (src->segment.format == GST_FORMAT_UNDEFINED) { + /* subclass must set valid format before calling this method */ + GST_WARNING_OBJECT (src, "segment format is not configured yet, ignore"); + GST_OBJECT_UNLOCK (src); + return FALSE; + } + + if (src->segment.format != segment->format) { + GST_WARNING_OBJECT (src, "segment format mismatched, ignore"); + GST_OBJECT_UNLOCK (src); + return FALSE; + } + + gst_segment_copy_into (segment, &src->segment); + + /* Mark pending segment. Will be sent before next data */ + src->priv->segment_pending = TRUE; + src->priv->segment_seqnum = gst_util_seqnum_next (); + + GST_DEBUG_OBJECT (src, "Starting new segment %" GST_SEGMENT_FORMAT, segment); + + GST_OBJECT_UNLOCK (src); + + return TRUE; +} + +static void +gst_base_idle_src_queue_object (GstBaseIdleSrc * src, GstMiniObject * obj) +{ + GST_LOG_OBJECT (src, "Queuing: %" GST_PTR_FORMAT, obj); + GST_OBJECT_LOCK (src); + g_queue_push_tail (src->priv->obj_queue, obj); + GST_OBJECT_UNLOCK (src); +} + +/** + * gst_base_idle_src_set_caps: + * @src: a #GstBaseIdleSrc + * @caps: (transfer none): a #GstCaps + * + * Set new caps on the src source pad. + * This *MUST* be called before submitting any buffers! + * + * Returns: %TRUE if the caps could be set + */ +gboolean +gst_base_idle_src_set_caps (GstBaseIdleSrc * src, GstCaps * caps) +{ + GstBaseIdleSrcClass *bclass; + gboolean res = TRUE; + GstCaps *current_caps; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + current_caps = gst_pad_get_current_caps (GST_BASE_IDLE_SRC_PAD (src)); + if (current_caps && gst_caps_is_equal (current_caps, caps)) { + GST_DEBUG_OBJECT (src, "New caps equal to old ones: %" GST_PTR_FORMAT, + caps); + res = TRUE; + } else { + if (bclass->set_caps) + res = bclass->set_caps (src, caps); + + if (res) { + gst_base_idle_src_queue_object (src, + (GstMiniObject *) gst_event_new_caps (caps)); + gst_base_idle_src_process_object_queue (src); + } + } + + if (current_caps) + gst_caps_unref (current_caps); + + return res; +} + +static GstCaps * +gst_base_idle_src_default_get_caps (GstBaseIdleSrc * bsrc, GstCaps * filter) +{ + GstCaps *caps = NULL; + GstPadTemplate *pad_template; + GstBaseIdleSrcClass *bclass; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (bsrc); + + pad_template = + gst_element_class_get_pad_template (GST_ELEMENT_CLASS (bclass), "src"); + + if (pad_template != NULL) { + caps = gst_pad_template_get_caps (pad_template); + + if (filter) { + GstCaps *intersection; + + intersection = + gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (caps); + caps = intersection; + } + } + return caps; +} + +static GstCaps * +gst_base_idle_src_default_fixate (GstBaseIdleSrc * bsrc, GstCaps * caps) +{ + GST_DEBUG_OBJECT (bsrc, "using default caps fixate function"); + return gst_caps_fixate (caps); +} + +static GstCaps * +gst_base_idle_src_fixate (GstBaseIdleSrc * bsrc, GstCaps * caps) +{ + GstBaseIdleSrcClass *bclass; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (bsrc); + + if (bclass->fixate) + caps = bclass->fixate (bsrc, caps); + + return caps; +} + +static gboolean +gst_base_idle_src_default_query (GstBaseIdleSrc * src, GstQuery * query) +{ + gboolean res; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_POSITION: + { + GstFormat format; + + gst_query_parse_position (query, &format, NULL); + + GST_DEBUG_OBJECT (src, "position query in format %s", + gst_format_get_name (format)); + + switch (format) { + case GST_FORMAT_PERCENT: + { + gint64 percent; + gint64 position; + gint64 duration; + + GST_OBJECT_LOCK (src); + position = src->segment.position; + duration = src->segment.duration; + GST_OBJECT_UNLOCK (src); + + if (position != -1 && duration != -1) { + if (position < duration) + percent = gst_util_uint64_scale (GST_FORMAT_PERCENT_MAX, position, + duration); + else + percent = GST_FORMAT_PERCENT_MAX; + } else + percent = -1; + + gst_query_set_position (query, GST_FORMAT_PERCENT, percent); + res = TRUE; + break; + } + default: + { + gint64 position; + GstFormat seg_format; + + GST_OBJECT_LOCK (src); + position = + gst_segment_to_stream_time (&src->segment, src->segment.format, + src->segment.position); + seg_format = src->segment.format; + GST_OBJECT_UNLOCK (src); + + if (position != -1) { + /* convert to requested format */ + res = + gst_pad_query_convert (src->srcpad, seg_format, + position, format, &position); + } else + res = TRUE; + + if (res) + gst_query_set_position (query, format, position); + + break; + } + } + break; + } + + case GST_QUERY_SEGMENT: + { + GstFormat format; + gint64 start, stop; + + GST_OBJECT_LOCK (src); + + format = src->segment.format; + + start = + gst_segment_to_stream_time (&src->segment, format, + src->segment.start); + if ((stop = src->segment.stop) == -1) + stop = src->segment.duration; + else + stop = gst_segment_to_stream_time (&src->segment, format, stop); + + gst_query_set_segment (query, src->segment.rate, format, start, stop); + + GST_OBJECT_UNLOCK (src); + res = TRUE; + break; + } + + case GST_QUERY_FORMATS: + { + gst_query_set_formats (query, 3, GST_FORMAT_DEFAULT, + GST_FORMAT_BYTES, GST_FORMAT_PERCENT); + res = TRUE; + break; + } + case GST_QUERY_CONVERT: + { + GstFormat src_fmt, dest_fmt; + gint64 src_val, dest_val; + + gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val); + + /* we can only convert between equal formats... */ + if (src_fmt == dest_fmt) { + dest_val = src_val; + res = TRUE; + } else + res = FALSE; + + gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val); + break; + } + case GST_QUERY_LATENCY: + { + GstClockTime min, max; + gboolean live; + + /* Subclasses should override and implement something useful */ + res = gst_base_idle_src_query_latency (src, &live, &min, &max); + + GST_LOG_OBJECT (src, "report latency: live %d, min %" GST_TIME_FORMAT + ", max %" GST_TIME_FORMAT, live, GST_TIME_ARGS (min), + GST_TIME_ARGS (max)); + + gst_query_set_latency (query, live, min, max); + break; + } + case GST_QUERY_JITTER: + case GST_QUERY_RATE: + res = FALSE; + break; + case GST_QUERY_BUFFERING: + { + GstFormat format, seg_format; + gint64 start, stop, estimated; + + gst_query_parse_buffering_range (query, &format, NULL, NULL, NULL); + + GST_DEBUG_OBJECT (src, "buffering query in format %s", + gst_format_get_name (format)); + + GST_OBJECT_LOCK (src); + estimated = -1; + start = -1; + stop = -1; + seg_format = src->segment.format; + GST_OBJECT_UNLOCK (src); + + /* convert to required format. When the conversion fails, we can't answer + * the query. When the value is unknown, we can don't perform conversion + * but report TRUE. */ + if (format != GST_FORMAT_PERCENT && stop != -1) { + res = gst_pad_query_convert (src->srcpad, seg_format, + stop, format, &stop); + } else { + res = TRUE; + } + if (res && format != GST_FORMAT_PERCENT && start != -1) + res = gst_pad_query_convert (src->srcpad, seg_format, + start, format, &start); + + gst_query_set_buffering_range (query, format, start, stop, estimated); + break; + } + case GST_QUERY_CAPS: + { + GstBaseIdleSrcClass *bclass; + GstCaps *caps, *filter; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + if (bclass->get_caps) { + gst_query_parse_caps (query, &filter); + if ((caps = bclass->get_caps (src, filter))) { + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + res = TRUE; + } else { + res = FALSE; + } + } else + res = FALSE; + break; + } + case GST_QUERY_URI:{ + if (GST_IS_URI_HANDLER (src)) { + gchar *uri = gst_uri_handler_get_uri (GST_URI_HANDLER (src)); + + if (uri != NULL) { + gst_query_set_uri (query, uri); + g_free (uri); + res = TRUE; + } else { + res = FALSE; + } + } else { + res = FALSE; + } + break; + } + default: + res = FALSE; + break; + } + GST_DEBUG_OBJECT (src, "query %s returns %d", GST_QUERY_TYPE_NAME (query), + res); + + return res; +} + +static gboolean +gst_base_idle_src_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstBaseIdleSrc *src; + GstBaseIdleSrcClass *bclass; + gboolean result = FALSE; + + src = GST_BASE_IDLE_SRC (parent); + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + if (bclass->query) + result = bclass->query (src, query); + + return result; +} + +static void +gst_base_idle_src_set_pool_flushing (GstBaseIdleSrc * src, gboolean flushing) +{ + GstBaseIdleSrcPrivate *priv = src->priv; + GstBufferPool *pool; + + GST_OBJECT_LOCK (src); + if ((pool = priv->pool)) + pool = gst_object_ref (pool); + GST_OBJECT_UNLOCK (src); + + if (pool) { + gst_buffer_pool_set_flushing (pool, flushing); + gst_object_unref (pool); + } +} + +/* all events send to this element directly. This is mainly done from the + * application. + */ +static gboolean +gst_base_idle_src_send_event (GstElement * element, GstEvent * event) +{ + GstBaseIdleSrc *src; + gboolean result = FALSE; + + src = GST_BASE_IDLE_SRC (element); + + GST_DEBUG_OBJECT (src, "handling event %p %" GST_PTR_FORMAT, event, event); + + switch (GST_EVENT_TYPE (event)) { + /* downstream serialized events */ + case GST_EVENT_EOS: + { + gst_base_idle_src_set_pool_flushing (src, TRUE); + event = NULL; + result = TRUE; + break; + } + case GST_EVENT_SEGMENT: + /* sending random SEGMENT downstream can break sync. */ + break; + case GST_EVENT_TAG: + case GST_EVENT_SINK_MESSAGE: + case GST_EVENT_CUSTOM_DOWNSTREAM: + case GST_EVENT_CUSTOM_BOTH: + case GST_EVENT_PROTECTION: + /* Insert TAG, CUSTOM_DOWNSTREAM, CUSTOM_BOTH, PROTECTION in the dataflow */ + gst_base_idle_src_queue_object (src, (GstMiniObject *) event); + event = NULL; + result = TRUE; + break; + case GST_EVENT_BUFFERSIZE: + /* does not seem to make much sense currently */ + break; + + /* upstream events */ + case GST_EVENT_QOS: + /* elements should override send_event and do something */ + break; + case GST_EVENT_NAVIGATION: + /* could make sense for elements that do something with navigation events + * but then they would need to override the send_event function */ + break; + case GST_EVENT_LATENCY: + /* does not seem to make sense currently */ + break; + + /* custom events */ + case GST_EVENT_CUSTOM_UPSTREAM: + /* override send_event if you want this */ + break; + case GST_EVENT_CUSTOM_DOWNSTREAM_OOB: + case GST_EVENT_CUSTOM_BOTH_OOB: + /* insert a random custom event into the pipeline */ + GST_DEBUG_OBJECT (src, "pushing custom OOB event downstream"); + result = gst_pad_push_event (src->srcpad, event); + /* we gave away the ref to the event in the push */ + event = NULL; + break; + default: + break; + } + + + return result; +} + +static void +gst_base_idle_src_update_qos (GstBaseIdleSrc * src, + gdouble proportion, GstClockTimeDiff diff, GstClockTime timestamp) +{ + GST_CAT_DEBUG_OBJECT (GST_CAT_QOS, src, + "qos: proportion: %lf, diff %" G_GINT64_FORMAT ", timestamp %" + GST_TIME_FORMAT, proportion, diff, GST_TIME_ARGS (timestamp)); + + GST_OBJECT_LOCK (src); + src->priv->proportion = proportion; + src->priv->earliest_time = timestamp + diff; + GST_OBJECT_UNLOCK (src); +} + + +static gboolean +gst_base_idle_src_default_event (GstBaseIdleSrc * src, GstEvent * event) +{ + gboolean result; + + GST_DEBUG_OBJECT (src, "handle event %" GST_PTR_FORMAT, event); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_QOS: + { + gdouble proportion; + GstClockTimeDiff diff; + GstClockTime timestamp; + + gst_event_parse_qos (event, NULL, &proportion, &diff, ×tamp); + gst_base_idle_src_update_qos (src, proportion, diff, timestamp); + result = TRUE; + break; + } + case GST_EVENT_RECONFIGURE: + result = TRUE; + break; + case GST_EVENT_LATENCY: + result = TRUE; + break; + default: + result = FALSE; + break; + } + return result; +} + +static gboolean +gst_base_idle_src_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + GstBaseIdleSrc *src; + GstBaseIdleSrcClass *bclass; + gboolean result = FALSE; + + src = GST_BASE_IDLE_SRC (parent); + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + if (bclass->event) { + if (!(result = bclass->event (src, event))) + goto subclass_failed; + } + +done: + gst_event_unref (event); + + return result; + + /* ERRORS */ +subclass_failed: + { + GST_DEBUG_OBJECT (src, "subclass refused event"); + goto done; + } +} + +static void +gst_base_idle_src_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstBaseIdleSrc *src; + + src = GST_BASE_IDLE_SRC (object); + + switch (prop_id) { + case PROP_DO_TIMESTAMP: + gst_base_idle_src_set_do_timestamp (src, g_value_get_boolean (value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_base_idle_src_get_property (GObject * object, guint prop_id, GValue * value, + GParamSpec * pspec) +{ + GstBaseIdleSrc *src; + + src = GST_BASE_IDLE_SRC (object); + + switch (prop_id) { + case PROP_DO_TIMESTAMP: + g_value_set_boolean (value, gst_base_idle_src_get_do_timestamp (src)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_base_idle_src_set_allocation (GstBaseIdleSrc * src, + GstBufferPool * pool, GstAllocator * allocator, + const GstAllocationParams * params) +{ + GstAllocator *oldalloc; + GstBufferPool *oldpool; + GstBaseIdleSrcPrivate *priv = src->priv; + + if (pool) { + GST_DEBUG_OBJECT (src, "activate pool"); + if (!gst_buffer_pool_set_active (pool, TRUE)) + goto activate_failed; + } + + GST_OBJECT_LOCK (src); + oldpool = priv->pool; + priv->pool = pool; + + oldalloc = priv->allocator; + priv->allocator = allocator; + + if (priv->pool) + gst_object_ref (priv->pool); + if (priv->allocator) + gst_object_ref (priv->allocator); + + if (params) + priv->params = *params; + else + gst_allocation_params_init (&priv->params); + GST_OBJECT_UNLOCK (src); + + if (oldpool) { + /* only deactivate if the pool is not the one we're using */ + if (oldpool != pool) { + GST_DEBUG_OBJECT (src, "deactivate old pool"); + gst_buffer_pool_set_active (oldpool, FALSE); + } + gst_object_unref (oldpool); + } + if (oldalloc) { + gst_object_unref (oldalloc); + } + return TRUE; + + /* ERRORS */ +activate_failed: + { + GST_ERROR_OBJECT (src, "failed to activate bufferpool."); + return FALSE; + } +} + +static gboolean +gst_base_idle_src_decide_allocation_default (GstBaseIdleSrc * src, + GstQuery * query) +{ + GstCaps *outcaps; + GstBufferPool *pool; + guint size, min, max; + GstAllocator *allocator; + GstAllocationParams params; + GstStructure *config; + gboolean update_allocator; + + gst_query_parse_allocation (query, &outcaps, NULL); + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params (query) > 0) { + /* try the allocator */ + gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); + update_allocator = TRUE; + } else { + allocator = NULL; + gst_allocation_params_init (¶ms); + update_allocator = FALSE; + } + + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + if (pool == NULL) { + /* no pool, we can make our own */ + GST_DEBUG_OBJECT (src, "no pool, making new pool"); + pool = gst_buffer_pool_new (); + } + } else { + pool = NULL; + size = min = max = 0; + } + + /* now configure */ + if (pool) { + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, outcaps, size, min, max); + gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + + /* buffer pool may have to do some changes */ + if (!gst_buffer_pool_set_config (pool, config)) { + config = gst_buffer_pool_get_config (pool); + + /* If change are not acceptable, fallback to generic pool */ + if (!gst_buffer_pool_config_validate_params (config, outcaps, size, min, + max)) { + GST_DEBUG_OBJECT (src, "unsupported pool, making new pool"); + + gst_object_unref (pool); + pool = gst_buffer_pool_new (); + gst_buffer_pool_config_set_params (config, outcaps, size, min, max); + gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + } + + if (!gst_buffer_pool_set_config (pool, config)) + goto config_failed; + } + } + + if (update_allocator) + gst_query_set_nth_allocation_param (query, 0, allocator, ¶ms); + else + gst_query_add_allocation_param (query, allocator, ¶ms); + if (allocator) + gst_object_unref (allocator); + + if (pool) { + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + gst_object_unref (pool); + } + + return TRUE; + +config_failed: + GST_ELEMENT_ERROR (src, RESOURCE, SETTINGS, + ("Failed to configure the buffer pool"), + ("Configuration is most likely invalid, please report this issue.")); + gst_object_unref (pool); + return FALSE; +} + +static gboolean +gst_base_idle_src_prepare_allocation (GstBaseIdleSrc * src, GstCaps * caps) +{ + GstBaseIdleSrcClass *bclass; + gboolean result = TRUE; + GstQuery *query; + GstBufferPool *pool = NULL; + GstAllocator *allocator = NULL; + GstAllocationParams params; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + /* make query and let peer pad answer, we don't really care if it worked or + * not, if it failed, the allocation query would contain defaults and the + * subclass would then set better values if needed */ + query = gst_query_new_allocation (caps, TRUE); + if (!gst_pad_peer_query (src->srcpad, query)) { + /* not a problem, just debug a little */ + GST_DEBUG_OBJECT (src, "peer ALLOCATION query failed"); + } + + g_assert (bclass->decide_allocation != NULL); + result = bclass->decide_allocation (src, query); + + GST_DEBUG_OBJECT (src, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result, + query); + + if (!result) + goto no_decide_allocation; + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params (query) > 0) { + gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); + } else { + allocator = NULL; + gst_allocation_params_init (¶ms); + } + + if (gst_query_get_n_allocation_pools (query) > 0) + gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL); + + result = gst_base_idle_src_set_allocation (src, pool, allocator, ¶ms); + + if (allocator) + gst_object_unref (allocator); + if (pool) + gst_object_unref (pool); + + gst_query_unref (query); + + return result; + + /* Errors */ +no_decide_allocation: + { + GST_WARNING_OBJECT (src, "Subclass failed to decide allocation"); + gst_query_unref (query); + + return result; + } +} + +/* default negotiation code. + * + * Take intersection between src and sink pads, take first + * caps and fixate. + + * only called from gst_base_idle_src_func + */ +static gboolean +gst_base_idle_src_default_negotiate (GstBaseIdleSrc * src) +{ + GstCaps *thiscaps; + GstCaps *caps = NULL; + GstCaps *peercaps = NULL; + gboolean result = FALSE; + + /* first see what is possible on our source pad */ + thiscaps = gst_pad_query_caps (GST_BASE_IDLE_SRC_PAD (src), NULL); + GST_DEBUG_OBJECT (src, "caps of src: %" GST_PTR_FORMAT, thiscaps); + /* nothing or anything is allowed, we're done */ + if (thiscaps == NULL || gst_caps_is_any (thiscaps)) + goto no_nego_needed; + + if (G_UNLIKELY (gst_caps_is_empty (thiscaps))) + goto no_caps; + + /* get the peer caps */ + peercaps = gst_pad_peer_query_caps (GST_BASE_IDLE_SRC_PAD (src), thiscaps); + GST_DEBUG_OBJECT (src, "caps of peer: %" GST_PTR_FORMAT, peercaps); + if (peercaps) { + /* The result is already a subset of our caps */ + caps = peercaps; + gst_caps_unref (thiscaps); + } else { + /* no peer, work with our own caps then */ + caps = thiscaps; + } + if (caps && !gst_caps_is_empty (caps)) { + /* now fixate */ + GST_DEBUG_OBJECT (src, "have caps: %" GST_PTR_FORMAT, caps); + if (gst_caps_is_any (caps)) { + GST_DEBUG_OBJECT (src, "any caps, we stop"); + /* hmm, still anything, so element can do anything and + * nego is not needed */ + result = TRUE; + } else { + caps = gst_base_idle_src_fixate (src, caps); + GST_DEBUG_OBJECT (src, "fixated to: %" GST_PTR_FORMAT, caps); + if (gst_caps_is_fixed (caps)) { + /* yay, fixed caps, use those then, it's possible that the subclass does + * not accept this caps after all and we have to fail. */ + result = gst_base_idle_src_set_caps (src, caps); + } + } + gst_caps_unref (caps); + } else { + if (caps) + gst_caps_unref (caps); + GST_DEBUG_OBJECT (src, "no common caps"); + } + return result; + +no_nego_needed: + { + GST_DEBUG_OBJECT (src, "no negotiation needed"); + if (thiscaps) + gst_caps_unref (thiscaps); + return TRUE; + } +no_caps: + { + GST_ELEMENT_ERROR (src, STREAM, FORMAT, + ("No supported formats found"), + ("This element did not produce valid caps")); + if (thiscaps) + gst_caps_unref (thiscaps); + return TRUE; + } +} + +/* only called from gst_base_idle_src_func */ +static gboolean +gst_base_idle_src_negotiate_unlocked (GstBaseIdleSrc * src) +{ + GstBaseIdleSrcClass *bclass; + gboolean result; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + GST_DEBUG_OBJECT (src, "starting negotiation"); + + if (G_LIKELY (bclass->negotiate)) + result = bclass->negotiate (src); + else + result = TRUE; + + if (G_LIKELY (result)) { + GstCaps *caps; + + caps = gst_pad_get_current_caps (src->srcpad); + + result = gst_base_idle_src_prepare_allocation (src, caps); + + if (caps) + gst_caps_unref (caps); + } + return result; +} + +/** + * gst_base_idle_src_get_buffer_pool: + * @src: a #GstBaseIdleSrc + * + * Returns: (nullable) (transfer full): the instance of the #GstBufferPool used + * by the src; unref it after usage. + */ +GstBufferPool * +gst_base_idle_src_get_buffer_pool (GstBaseIdleSrc * src) +{ + GstBufferPool *ret = NULL; + + g_return_val_if_fail (GST_IS_BASE_IDLE_SRC (src), NULL); + + GST_OBJECT_LOCK (src); + if (src->priv->pool) + ret = gst_object_ref (src->priv->pool); + GST_OBJECT_UNLOCK (src); + + return ret; +} + +/** + * gst_base_idle_src_get_allocator: + * @src: a #GstBaseIdleSrc + * @allocator: (out) (optional) (nullable) (transfer full): the #GstAllocator + * used + * @params: (out caller-allocates) (optional): the #GstAllocationParams of @allocator + * + * Lets #GstBaseIdleSrc sub-classes to know the memory @allocator + * used by the base class and its @params. + * + * Unref the @allocator after usage. + */ +void +gst_base_idle_src_get_allocator (GstBaseIdleSrc * src, + GstAllocator ** allocator, GstAllocationParams * params) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + + GST_OBJECT_LOCK (src); + if (allocator) + *allocator = src->priv->allocator ? + gst_object_ref (src->priv->allocator) : NULL; + + if (params) + *params = src->priv->params; + GST_OBJECT_UNLOCK (src); +} + +static void +gst_base_idle_src_add_timestamp (GstBaseIdleSrc * src, GstBuffer * buf) +{ + GstClock *clock; + GstClockTime base_time; + GstClockTime now; + GstClockTime running_time; + + if ((clock = GST_ELEMENT_CLOCK (src)) == NULL) { + GST_ERROR_OBJECT (src, "No clock!?! Can't do timestamp"); + return; + } else { + gst_object_ref (clock); + } + base_time = GST_ELEMENT_CAST (src)->base_time; + now = gst_clock_get_time (clock); + gst_object_unref (clock); + + running_time = now - base_time; + GST_BUFFER_PTS (buf) = running_time; + GST_BUFFER_DTS (buf) = running_time; +} + +static void +gst_base_idle_src_process_object (GstBaseIdleSrc * src, GstMiniObject * obj) +{ + GstPad *pad = src->srcpad; + + GST_PAD_STREAM_LOCK (pad); + + if (GST_IS_BUFFER (obj)) { + GstBuffer *buf = GST_BUFFER_CAST (obj); + GstFlowReturn ret; + + if (src->priv->do_timestamp) { + gst_base_idle_src_add_timestamp (src, buf); + } + + GST_DEBUG_OBJECT (src, "About to push Buffer %" GST_PTR_FORMAT, buf); + + ret = gst_pad_push (pad, buf); + if (ret != GST_FLOW_OK) + GST_ERROR ("Got ret: %s", gst_flow_get_name (ret)); + } else if (GST_IS_EVENT (obj)) { + GstEvent *event = GST_EVENT_CAST (obj); + gboolean ret; + + GST_DEBUG_OBJECT (src, "About to push Event %" GST_PTR_FORMAT, event); + ret = gst_pad_push_event (pad, event); + if (!ret) + GST_ERROR ("HUUUUAA"); + } else if (GST_IS_CAPS (obj)) { + GST_DEBUG_OBJECT (src, "About to push Caps %" GST_PTR_FORMAT, obj); + } + + GST_PAD_STREAM_UNLOCK (pad); +} + +static void +gst_base_idle_src_process_object_queue (GstBaseIdleSrc * src) +{ + GstMiniObject *obj; + GST_OBJECT_LOCK (src); + while ((obj = g_queue_pop_head (src->priv->obj_queue))) { + + GST_OBJECT_UNLOCK (src); + gst_base_idle_src_process_object (src, obj); + GST_OBJECT_LOCK (src); + } + GST_OBJECT_UNLOCK (src); +} + +static gpointer +gst_base_idle_src_func (gpointer user_data) +{ + GstBaseIdleSrc *src = GST_BASE_IDLE_SRC (user_data); + + GST_DEBUG_OBJECT (src, "Thread func started"); + + GST_PAD_STREAM_LOCK (src->srcpad); + if (gst_pad_check_reconfigure (src->srcpad)) { + if (!gst_base_idle_src_negotiate_unlocked (src)) { + GST_ELEMENT_FLOW_ERROR (src, GST_FLOW_NOT_NEGOTIATED); + gst_pad_push_event (src->srcpad, gst_event_new_eos ()); + GST_PAD_STREAM_UNLOCK (src->srcpad); + return NULL; + } + } + GST_PAD_STREAM_UNLOCK (src->srcpad); + + gst_base_idle_src_process_object_queue (src); + + return NULL; +} + +static void +gst_base_idle_src_start_task (GstBaseIdleSrc * src, gboolean wait) +{ + GST_DEBUG_OBJECT (src, "Starting Task"); + /* if we already have an outstanding task, join it */ + if (src->priv->thread) + g_thread_join (src->priv->thread); + + src->priv->thread = + g_thread_new (GST_ELEMENT_NAME (src), gst_base_idle_src_func, src); + if (wait) { + g_thread_join (src->priv->thread); + src->priv->thread = NULL; + } +} + +static void +gst_base_idle_src_check_pending_segment (GstBaseIdleSrc * src) +{ + GST_DEBUG_OBJECT (src, "Checking pending segment"); + /* push events to close/start our segment before we push the buffer. */ + if (G_UNLIKELY (src->priv->segment_pending)) { + GstEvent *seg_event = gst_event_new_segment (&src->segment); + + gst_event_set_seqnum (seg_event, src->priv->segment_seqnum); + src->priv->segment_seqnum = gst_util_seqnum_next (); + gst_base_idle_src_queue_object (src, (GstMiniObject *) seg_event); + GST_DEBUG_OBJECT (src, "Queing segment event %" GST_PTR_FORMAT, seg_event); + + src->priv->segment_pending = FALSE; + } +} + +static void +gst_base_idle_src_add_stream_start (GstBaseIdleSrc * src) +{ + gchar *stream_id; + GstEvent *event; + + stream_id = + gst_pad_create_stream_id (src->srcpad, GST_ELEMENT_CAST (src), NULL); + + GST_DEBUG_OBJECT (src, "Pushing STREAM_START"); + event = gst_event_new_stream_start (stream_id); + gst_event_set_group_id (event, gst_util_group_id_next ()); + + gst_base_idle_src_queue_object (src, (GstMiniObject *) event); + g_free (stream_id); +} + +static gboolean +gst_base_idle_src_start (GstBaseIdleSrc * src) +{ + GstBaseIdleSrcClass *bclass; + gboolean result; + + GST_DEBUG_OBJECT (src, "Starting"); + + GST_OBJECT_LOCK (src); + + gst_segment_init (&src->segment, src->segment.format); + GST_OBJECT_UNLOCK (src); + + src->running = TRUE; + src->priv->segment_pending = TRUE; + src->priv->segment_seqnum = gst_util_seqnum_next (); + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + if (bclass->start) + result = bclass->start (src); + else + result = TRUE; + + if (!result) + goto could_not_start; + + gst_base_idle_src_add_stream_start (src); + gst_base_idle_src_start_task (src, TRUE); + + return result; + +could_not_start: + { + GST_DEBUG_OBJECT (src, "could not start"); + /* subclass is supposed to post a message but we post one as a fallback + * just in case. We don't have to call _stop. */ + GST_ELEMENT_ERROR (src, CORE, STATE_CHANGE, (NULL), ("Failed to start")); + return FALSE; + } +} + +static gboolean +gst_base_idle_src_stop (GstBaseIdleSrc * src) +{ + GstBaseIdleSrcClass *bclass; + GstMiniObject *obj; + gboolean result = TRUE; + + GST_DEBUG_OBJECT (src, "stopping source"); + + src->running = FALSE; + if (src->priv->thread) { + g_thread_join (src->priv->thread); + src->priv->thread = NULL; + } + + /* clean up any leftovers on the queue */ + while ((obj = g_queue_pop_head (src->priv->obj_queue))) { + gst_mini_object_unref (obj); + } + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + if (bclass->stop) + result = bclass->stop (src); + + gst_base_idle_src_set_allocation (src, NULL, NULL, NULL); + + return result; +} + +static gboolean +gst_base_idle_src_activate_push (GstPad * pad, GstObject * parent, + gboolean active) +{ + GstBaseIdleSrc *basesrc; + + basesrc = GST_BASE_IDLE_SRC (parent); + + /* prepare subclass first */ + if (active) { + GST_DEBUG_OBJECT (basesrc, "Activating in push mode"); + + if (G_UNLIKELY (!gst_base_idle_src_start (basesrc))) + goto error_start; + } else { + GST_DEBUG_OBJECT (basesrc, "Deactivating in push mode"); + /* now we can stop the source */ + if (G_UNLIKELY (!gst_base_idle_src_stop (basesrc))) + goto error_stop; + } + return TRUE; + +error_start: + { + GST_WARNING_OBJECT (basesrc, "Failed to start in push mode"); + return FALSE; + } +error_stop: + { + GST_DEBUG_OBJECT (basesrc, "Failed to stop in push mode"); + return FALSE; + } +} + +static gboolean +gst_base_idle_src_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + gboolean res; + + GST_DEBUG_OBJECT (pad, "activating in mode %d", mode); + + switch (mode) { + case GST_PAD_MODE_PULL: + g_assert_not_reached (); + break; + case GST_PAD_MODE_PUSH: + res = gst_base_idle_src_activate_push (pad, parent, active); + break; + default: + GST_LOG_OBJECT (pad, "unknown activation mode %d", mode); + res = FALSE; + break; + } + return res; +} + + +/** + * gst_base_idle_src_submit_buffer: + * @src: a #GstBaseIdleSrc + * @buffer: (transfer full): a #GstBuffer + * + * Subclasses can call this to submit a buffer to be pushed out later. + * + * Since: 1.22 + */ +void +gst_base_idle_src_submit_buffer (GstBaseIdleSrc * src, GstBuffer * buffer) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + g_return_if_fail (GST_IS_BUFFER (buffer)); + + if (!src->running) { + GST_ERROR_OBJECT (src, "Sending buffer to stopped src is not valid"); + gst_buffer_unref (buffer); + return; + } + + gst_base_idle_src_check_pending_segment (src); + + /* we need it to be writable later in get_range() where we use get_writable */ + gst_base_idle_src_queue_object (src, (GstMiniObject *) buffer); + + gst_base_idle_src_start_task (src, FALSE); +} + +/** + * gst_base_idle_src_submit_buffer_list: + * @src: a #GstBaseIdleSrc + * @buffer_list: (transfer full): a #GstBufferList + * + * Subclasses can call this to submit a buffer list to be pushed out later. + * + * Since: 1.22 + */ +void +gst_base_idle_src_submit_buffer_list (GstBaseIdleSrc * src, + GstBufferList * buffer_list) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + g_return_if_fail (GST_IS_BUFFER_LIST (buffer_list)); + + if (!src->running) { + GST_ERROR_OBJECT (src, "Sending bufferlist to stopped src is not valid"); + gst_buffer_list_unref (buffer_list); + return; + } + + gst_base_idle_src_check_pending_segment (src); + + /* we need it to be writable later in get_range() where we use get_writable */ + gst_base_idle_src_queue_object (src, (GstMiniObject *) buffer_list); + + GST_LOG_OBJECT (src, "%u buffers submitted in buffer list", + gst_buffer_list_length (buffer_list)); + + gst_base_idle_src_start_task (src, FALSE); +} + + +/** + * gst_base_idle_src_alloc_buffer: + * @src: a #GstBaseIdleSrc + * @size: a guint with the size of the buffer + * @buffer: (transfer full): a #GstBuffer + * + * Subclasses can call this to alloc a buffer. + * + * Since: 1.22 + */ +GstFlowReturn +gst_base_idle_src_alloc_buffer (GstBaseIdleSrc * src, + guint size, GstBuffer ** buffer) +{ + GstFlowReturn ret; + GstBaseIdleSrcPrivate *priv = src->priv; + GstBufferPool *pool = NULL; + GstAllocator *allocator = NULL; + GstAllocationParams params; + + GST_OBJECT_LOCK (src); + if (priv->pool) { + pool = gst_object_ref (priv->pool); + } else if (priv->allocator) { + allocator = gst_object_ref (priv->allocator); + } + params = priv->params; + GST_OBJECT_UNLOCK (src); + + if (pool) { + ret = gst_buffer_pool_acquire_buffer (pool, buffer, NULL); + } else if (size != -1) { + *buffer = gst_buffer_new_allocate (allocator, size, ¶ms); + if (G_UNLIKELY (*buffer == NULL)) + goto alloc_failed; + + ret = GST_FLOW_OK; + } else { + GST_WARNING_OBJECT (src, "Not trying to alloc %u bytes. Blocksize not set?", + size); + goto alloc_failed; + } + +done: + if (pool) + gst_object_unref (pool); + if (allocator) + gst_object_unref (allocator); + + return ret; + + /* ERRORS */ +alloc_failed: + { + GST_ERROR_OBJECT (src, "Failed to allocate %u bytes", size); + ret = GST_FLOW_ERROR; + goto done; + } +} + +static void +gst_base_idle_src_finalize (GObject * object) +{ + GstBaseIdleSrc *src; + src = GST_BASE_IDLE_SRC (object); + + g_queue_free (src->priv->obj_queue); + /* FIXME: empty this queue potentially... */ + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_base_idle_src_class_init (GstBaseIdleSrcClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + + gobject_class = G_OBJECT_CLASS (klass); + gstelement_class = GST_ELEMENT_CLASS (klass); + + if (private_offset != 0) + g_type_class_adjust_private_offset (klass, &private_offset); + + GST_DEBUG_CATEGORY_INIT (gst_base_idle_src_debug, "idlesrc", 0, + "idlesrc element"); + + parent_class = g_type_class_peek_parent (klass); + + gobject_class->finalize = gst_base_idle_src_finalize; + gobject_class->set_property = gst_base_idle_src_set_property; + gobject_class->get_property = gst_base_idle_src_get_property; + + g_object_class_install_property (gobject_class, PROP_DO_TIMESTAMP, + g_param_spec_boolean ("do-timestamp", "Do timestamp", + "Apply current stream time to buffers", DEFAULT_DO_TIMESTAMP, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gstelement_class->send_event = + GST_DEBUG_FUNCPTR (gst_base_idle_src_send_event); + + klass->get_caps = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_get_caps); + klass->negotiate = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_negotiate); + klass->fixate = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_fixate); + klass->query = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_query); + klass->event = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_event); + klass->decide_allocation = + GST_DEBUG_FUNCPTR (gst_base_idle_src_decide_allocation_default); + + /* Registering debug symbols for function pointers */ + + GST_DEBUG_REGISTER_FUNCPTR (gst_base_idle_src_event); + GST_DEBUG_REGISTER_FUNCPTR (gst_base_idle_src_query); + GST_DEBUG_REGISTER_FUNCPTR (gst_base_idle_src_fixate); +} + +static void +gst_base_idle_src_init (GstBaseIdleSrc * src, gpointer g_class) +{ + GstPad *pad; + GstPadTemplate *pad_template; + + src->priv = gst_base_idle_src_get_instance_private (src); + + pad_template = + gst_element_class_get_pad_template (GST_ELEMENT_CLASS (g_class), "src"); + g_return_if_fail (pad_template != NULL); + + GST_DEBUG_OBJECT (src, "creating src pad"); + pad = gst_pad_new_from_template (pad_template, "src"); + + GST_DEBUG_OBJECT (src, "setting functions on src pad"); + gst_pad_set_activatemode_function (pad, gst_base_idle_src_activate_mode); + gst_pad_set_event_function (pad, gst_base_idle_src_event); + gst_pad_set_query_function (pad, gst_base_idle_src_query); + + /* hold pointer to pad */ + src->srcpad = pad; + GST_DEBUG_OBJECT (src, "adding src pad"); + gst_element_add_pad (GST_ELEMENT (src), pad); + + /* we operate in BYTES by default */ + gst_base_idle_src_set_format (src, GST_FORMAT_BYTES); + src->priv->do_timestamp = DEFAULT_DO_TIMESTAMP; + + GST_OBJECT_FLAG_SET (src, GST_ELEMENT_FLAG_SOURCE); + + src->priv->obj_queue = g_queue_new (); + + GST_DEBUG_OBJECT (src, "init done"); +} + +GType +gst_base_idle_src_get_type (void) +{ + static gsize base_idle_src_type = 0; + + if (g_once_init_enter (&base_idle_src_type)) { + GType _type; + static const GTypeInfo base_idle_src_info = { + sizeof (GstBaseIdleSrcClass), + NULL, + NULL, + (GClassInitFunc) gst_base_idle_src_class_init, + NULL, + NULL, + sizeof (GstBaseIdleSrc), + 0, + (GInstanceInitFunc) gst_base_idle_src_init, + }; + + _type = g_type_register_static (GST_TYPE_ELEMENT, + "GstBaseIdleSrc", &base_idle_src_info, G_TYPE_FLAG_ABSTRACT); + + private_offset = + g_type_add_instance_private (_type, sizeof (GstBaseIdleSrcPrivate)); + + g_once_init_leave (&base_idle_src_type, _type); + } + return base_idle_src_type; +} diff --git a/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h new file mode 100644 index 00000000000..142589909ed --- /dev/null +++ b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h @@ -0,0 +1,220 @@ +/* GStreamer + * Copyright (C) 1999,2000 Erik Walthinsen + * 2000 Wim Taymans + * 2005 Wim Taymans + * + * gstbaseidlesrc.h: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_BASE_IDLE_SRC_H__ +#define __GST_BASE_IDLE_SRC_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_BASE_IDLE_SRC (gst_base_idle_src_get_type()) +#define GST_BASE_IDLE_SRC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_BASE_IDLE_SRC,GstBaseIdleSrc)) +#define GST_BASE_IDLE_SRC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_BASE_IDLE_SRC,GstBaseIdleSrcClass)) +#define GST_BASE_IDLE_SRC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_BASE_IDLE_SRC, GstBaseIdleSrcClass)) +#define GST_IS_BASE_IDLE_SRC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_BASE_IDLE_SRC)) +#define GST_IS_BASE_IDLE_SRC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_BASE_IDLE_SRC)) +#define GST_BASE_IDLE_SRC_CAST(obj) ((GstBaseIdleSrc *)(obj)) + +typedef struct _GstBaseIdleSrc GstBaseIdleSrc; +typedef struct _GstBaseIdleSrcClass GstBaseIdleSrcClass; +typedef struct _GstBaseIdleSrcPrivate GstBaseIdleSrcPrivate; + +/** + * GST_BASE_IDLE_SRC_PAD: + * @obj: base source instance + * + * Gives the pointer to the #GstPad object of the element. + */ +#define GST_BASE_IDLE_SRC_PAD(obj) (GST_BASE_IDLE_SRC_CAST (obj)->srcpad) + +/** + * GstBaseIdleSrc: + * + * The opaque #GstBaseIdleSrc data structure. + */ +struct _GstBaseIdleSrc { + GstElement element; + + /*< protected >*/ + GstPad *srcpad; + + gboolean is_live; + + /* MT-protected (with STREAM_LOCK *and* OBJECT_LOCK) */ + GstSegment segment; + /* MT-protected (with STREAM_LOCK) */ + gboolean need_newsegment; + + gint num_buffers; + gint num_buffers_left; + + gboolean running; + + GstBaseIdleSrcPrivate *priv; + + /*< private >*/ + gpointer _gst_reserved[GST_PADDING_LARGE]; +}; + +/** + * GstBaseIdleSrcClass: + * @parent_class: Element parent class + * @get_caps: Called to get the caps to report + * @negotiate: Negotiated the caps with the peer. + * @fixate: Called during negotiation if caps need fixating. Implement instead of + * setting a fixate function on the source pad. + * @set_caps: Notify subclass of changed output caps + * @decide_allocation: configure the allocation query + * @start: Start processing. Subclasses should open resources and prepare + * to produce data. Implementation should call gst_base_idle_src_start_complete() + * when the operation completes, either from the current thread or any other + * thread that finishes the start operation asynchronously. + * @stop: Stop processing. Subclasses should use this to close resources. + * @get_size: Return the total size of the resource, in the format set by + * gst_base_idle_src_set_format(). + * @query: Handle a requested query. + * @event: Override this to implement custom event handling. + * @alloc: Ask the subclass to allocate a buffer with for offset and size. The + * default implementation will create a new buffer from the negotiated allocator. + * @fill: Ask the subclass to fill the buffer with data for offset and size. The + * passed buffer is guaranteed to hold the requested amount of bytes. + * + * Subclasses can override any of the available virtual methods or not, as + * needed. At the minimum, the @create method should be overridden to produce + * buffers. + */ +struct _GstBaseIdleSrcClass { + GstElementClass parent_class; + + /*< public >*/ + /* virtual methods for subclasses */ + + /** + * GstBaseIdleSrcClass::get_caps: + * @filter: (in) (nullable): + * + * Called to get the caps to report. + */ + GstCaps* (*get_caps) (GstBaseIdleSrc *src, GstCaps *filter); + /* decide on caps */ + gboolean (*negotiate) (GstBaseIdleSrc *src); + /* called if, in negotiation, caps need fixating */ + GstCaps * (*fixate) (GstBaseIdleSrc *src, GstCaps *caps); + /* notify the subclass of new caps */ + gboolean (*set_caps) (GstBaseIdleSrc *src, GstCaps *caps); + + /* setup allocation query */ + gboolean (*decide_allocation) (GstBaseIdleSrc *src, GstQuery *query); + + /* start and stop processing, ideal for opening/closing the resource */ + gboolean (*start) (GstBaseIdleSrc *src); + gboolean (*stop) (GstBaseIdleSrc *src); + + /** + * GstBaseIdleSrcClass::get_size: + * @size: (out): + * + * Get the total size of the resource in the format set by + * gst_base_idle_src_set_format(). + * + * Returns: %TRUE if the size is available and has been set. + */ + gboolean (*get_size) (GstBaseIdleSrc *src, guint64 *size); + + /* notify subclasses of a query */ + gboolean (*query) (GstBaseIdleSrc *src, GstQuery *query); + + /* notify subclasses of an event */ + gboolean (*event) (GstBaseIdleSrc *src, GstEvent *event); + + + /*< private >*/ + gpointer _gst_reserved[GST_PADDING_LARGE]; +}; + +GST_BASE_API +GType gst_base_idle_src_get_type (void); + +GST_BASE_API +void gst_base_idle_src_set_live (GstBaseIdleSrc *src, gboolean live); + +GST_BASE_API +gboolean gst_base_idle_src_is_live (GstBaseIdleSrc *src); + +GST_BASE_API +void gst_base_idle_src_set_format (GstBaseIdleSrc *src, GstFormat format); + +GST_BASE_API +void gst_base_idle_src_set_automatic_eos (GstBaseIdleSrc * src, gboolean automatic_eos); + + +GST_BASE_API +gboolean gst_base_idle_src_negotiate (GstBaseIdleSrc *src); + + +GST_BASE_API +gboolean gst_base_idle_src_query_latency (GstBaseIdleSrc *src, gboolean * live, + GstClockTime * min_latency, + GstClockTime * max_latency); + +GST_BASE_API +void gst_base_idle_src_set_do_timestamp (GstBaseIdleSrc *src, gboolean timestamp); + +GST_BASE_API +gboolean gst_base_idle_src_get_do_timestamp (GstBaseIdleSrc *src); + +GST_BASE_API +gboolean gst_base_idle_src_new_segment (GstBaseIdleSrc *src, + const GstSegment * segment); + +GST_BASE_API +gboolean gst_base_idle_src_set_caps (GstBaseIdleSrc *src, GstCaps *caps); + +GST_BASE_API +GstBufferPool * gst_base_idle_src_get_buffer_pool (GstBaseIdleSrc *src); + +GST_BASE_API +void gst_base_idle_src_get_allocator (GstBaseIdleSrc *src, + GstAllocator **allocator, + GstAllocationParams *params); + +GST_BASE_API +void gst_base_idle_src_submit_buffer (GstBaseIdleSrc * src, + GstBuffer * buffer); + +GST_BASE_API +void gst_base_idle_src_submit_buffer_list (GstBaseIdleSrc * src, + GstBufferList * buffer_list); + +GST_BASE_API +GstFlowReturn gst_base_idle_src_alloc_buffer (GstBaseIdleSrc * src, + guint size, + GstBuffer ** buffer); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstBaseIdleSrc, gst_object_unref) + +G_END_DECLS + +#endif /* __GST_BASE_IDLE_SRC_H__ */ diff --git a/subprojects/gstreamer/libs/gst/base/meson.build b/subprojects/gstreamer/libs/gst/base/meson.build index 9c85dd86166..64292f15b89 100644 --- a/subprojects/gstreamer/libs/gst/base/meson.build +++ b/subprojects/gstreamer/libs/gst/base/meson.build @@ -4,6 +4,7 @@ gst_base_sources = files( 'gstbaseparse.c', 'gstbasesink.c', 'gstbasesrc.c', + 'gstbaseidlesrc.c', 'gstbasetransform.c', 'gstbitreader.c', 'gstbitwriter.c', @@ -26,6 +27,7 @@ gst_base_headers = files( 'gstbaseparse.h', 'gstbasesink.h', 'gstbasesrc.h', + 'gstbaseidlesrc.h', 'gstbasetransform.h', 'gstbitreader.h', 'gstbitwriter.h', @@ -101,6 +103,7 @@ install_headers('base.h', 'gstbaseparse.h', 'gstbasesink.h', 'gstbasesrc.h', + 'gstbaseidlesrc.h', 'gstbasetransform.h', 'gstbitreader.h', 'gstbitwriter.h', diff --git a/subprojects/gstreamer/tests/check/libs/baseidlesrc.c b/subprojects/gstreamer/tests/check/libs/baseidlesrc.c new file mode 100644 index 00000000000..7d7aa1a4a39 --- /dev/null +++ b/subprojects/gstreamer/tests/check/libs/baseidlesrc.c @@ -0,0 +1,112 @@ +/* GStreamer + * + * some unit tests for GstBaseIdleSrc + * + * Copyright (C) 2023 Havard Graff + * 2023 Camilo Celis Guzman + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include + +typedef GstBaseIdleSrc TestIdleSrc; +typedef GstBaseIdleSrcClass TestIdleSrcClass; + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GType test_idle_src_get_type (void); + +G_DEFINE_TYPE (TestIdleSrc, test_idle_src, GST_TYPE_BASE_IDLE_SRC); + +static void +test_idle_src_init (TestIdleSrc * src) +{ +} + +static void +test_idle_src_class_init (TestIdleSrcClass * klass) +{ + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &src_template); +} + +GST_START_TEST (baseidlesrc_up_and_down) +{ + GstElement *src; + GstHarness *h; + + src = g_object_new (test_idle_src_get_type (), NULL); + + h = gst_harness_new_with_element (GST_ELEMENT (src), NULL, "src"); + + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (baseidlesrc_submit_buffer) +{ + GstElement *src; + GstHarness *h; + GstBaseIdleSrc *base_src; + GstBuffer *buf; + guint i; + + src = g_object_new (test_idle_src_get_type (), NULL); + base_src = GST_BASE_IDLE_SRC (src); + + h = gst_harness_new_with_element (GST_ELEMENT (src), NULL, "src"); + + gst_harness_set_sink_caps_str (h, "foo/bar"); + gst_harness_play (h); + + for (i = 0; i < 5; i++) { + fail_unless_equals_int (GST_FLOW_OK, + gst_base_idle_src_alloc_buffer (base_src, 100, &buf)); + gst_base_idle_src_submit_buffer (base_src, buf); + gst_buffer_unref (gst_harness_pull (h)); + } + + gst_harness_teardown (h); +} + +GST_END_TEST; + +static Suite * +baseidlesrc_suite (void) +{ + Suite *s = suite_create ("GstBaseIdleSrc"); + TCase *tc = tcase_create ("general"); + + suite_add_tcase (s, tc); + tcase_add_test (tc, baseidlesrc_up_and_down); + tcase_add_test (tc, baseidlesrc_submit_buffer); + + return s; +} + +GST_CHECK_MAIN (baseidlesrc); diff --git a/subprojects/gstreamer/tests/check/libs/basesrc.c b/subprojects/gstreamer/tests/check/libs/basesrc.c index 253d69ab64b..df863424434 100644 --- a/subprojects/gstreamer/tests/check/libs/basesrc.c +++ b/subprojects/gstreamer/tests/check/libs/basesrc.c @@ -276,7 +276,7 @@ GST_END_TEST; * - makes sure source doesn't send EOS event when reaching the max. * number of buffers configured in pull-mode * - make sure source doesn't send EOS event either when being shut down - * (PAUSED => READY state change) after EOSing in pull mode + * (PAUSED => READY state change) after EOSing in pull mode */ GST_START_TEST (basesrc_eos_events_pull) { diff --git a/subprojects/gstreamer/tests/check/meson.build b/subprojects/gstreamer/tests/check/meson.build index 4e303df5358..8c94caf2796 100644 --- a/subprojects/gstreamer/tests/check/meson.build +++ b/subprojects/gstreamer/tests/check/meson.build @@ -57,6 +57,7 @@ core_tests = [ [ 'libs/baseparse.c' ], [ 'libs/basesrc.c', not gst_registry ], [ 'libs/basesink.c', not gst_registry ], + [ 'libs/baseidlesrc.c', not gst_registry ], [ 'libs/bitreader.c' ], [ 'libs/bitwriter.c' ], [ 'libs/bytereader.c' ], From b6d7ddf8eaa0e91e7ac867a597e4fcba47894648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Graff?= Date: Sat, 2 Dec 2023 19:08:20 +0100 Subject: [PATCH 333/339] baseidlesrc: FIXUP gsize instead of guint --- subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c | 4 ++-- subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c index 394c6f39253..01192fc82fe 100644 --- a/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c +++ b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c @@ -1508,7 +1508,7 @@ gst_base_idle_src_submit_buffer_list (GstBaseIdleSrc * src, /** * gst_base_idle_src_alloc_buffer: * @src: a #GstBaseIdleSrc - * @size: a guint with the size of the buffer + * @size: a gsize with the size of the buffer * @buffer: (transfer full): a #GstBuffer * * Subclasses can call this to alloc a buffer. @@ -1517,7 +1517,7 @@ gst_base_idle_src_submit_buffer_list (GstBaseIdleSrc * src, */ GstFlowReturn gst_base_idle_src_alloc_buffer (GstBaseIdleSrc * src, - guint size, GstBuffer ** buffer) + gsize size, GstBuffer ** buffer) { GstFlowReturn ret; GstBaseIdleSrcPrivate *priv = src->priv; diff --git a/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h index 142589909ed..cf9fea5660a 100644 --- a/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h +++ b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h @@ -210,7 +210,7 @@ void gst_base_idle_src_submit_buffer_list (GstBaseIdleSrc * src, GST_BASE_API GstFlowReturn gst_base_idle_src_alloc_buffer (GstBaseIdleSrc * src, - guint size, + gsize size, GstBuffer ** buffer); G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstBaseIdleSrc, gst_object_unref) From 149f565325b8a1bf4f172b304fe07416fb7ed5fb Mon Sep 17 00:00:00 2001 From: Tulio Beloqui Date: Mon, 4 Dec 2023 13:02:38 +0100 Subject: [PATCH 334/339] mfvideosrc: added logic to read a frame to verify the src can access the selected camera --- .../sys/mediafoundation/gstmfsourcereader.cpp | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfsourcereader.cpp b/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfsourcereader.cpp index f0066e3995f..844e328e41a 100644 --- a/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfsourcereader.cpp +++ b/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfsourcereader.cpp @@ -117,6 +117,9 @@ static gboolean gst_mf_source_enum_device_activate (GstMFSourceReader * self, GstMFSourceType source_type, GList ** device_activates); static void gst_mf_device_activate_free (GstMFDeviceActivate * activate); +static GstFlowReturn +gst_mf_source_reader_read_sample (GstMFSourceReader * self); + #define gst_mf_source_reader_parent_class parent_class G_DEFINE_TYPE (GstMFSourceReader, gst_mf_source_reader, GST_TYPE_MF_SOURCE_OBJECT); @@ -283,6 +286,7 @@ compare_caps_func (gconstpointer a, gconstpointer b) static gboolean gst_mf_source_reader_open (GstMFSourceReader * self, IMFActivate * activate) { + GstMFSourceObject *object = GST_MF_SOURCE_OBJECT (self); GList *iter; HRESULT hr; ComPtr < IMFSourceReader > reader; @@ -332,6 +336,24 @@ gst_mf_source_reader_open (GstMFSourceReader * self, IMFActivate * activate) GST_DEBUG_OBJECT (self, "Available output caps %" GST_PTR_FORMAT, self->supported_caps); + /* if we have specified a device path, try to read a frame, this will return + an error if the device is busy. + Note: we only want to do this when a specific device is specified, as + this code is also used for camera enumeration */ + if (object->device_path != nullptr) { + if (!gst_mf_source_reader_set_caps (GST_MF_SOURCE_OBJECT (self), self->supported_caps)) { + gst_mf_source_reader_close (self); + return FALSE; + } + + if (gst_mf_source_reader_read_sample (self) != GST_FLOW_OK){ + gst_mf_source_reader_close (self); + return FALSE; + } + + gst_queue_array_clear (self->queue); + } + return TRUE; } From ae2f629311cc2436646978c0dab2fdabb435df54 Mon Sep 17 00:00:00 2001 From: Knut Saastad Date: Fri, 8 Dec 2023 14:42:59 +0100 Subject: [PATCH 335/339] gstwasapi: add device is-default info. --- .../sys/wasapi/gstwasapiutil.c | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-bad/sys/wasapi/gstwasapiutil.c b/subprojects/gst-plugins-bad/sys/wasapi/gstwasapiutil.c index 8f935837eea..6683c82653b 100644 --- a/subprojects/gst-plugins-bad/sys/wasapi/gstwasapiutil.c +++ b/subprojects/gst-plugins-bad/sys/wasapi/gstwasapiutil.c @@ -385,6 +385,24 @@ gst_wasapi_util_get_devices (GstMMDeviceEnumerator * self, if (!enum_handle) return FALSE; + IMMDevice *default_render_device = NULL; + wchar_t * default_render_id = NULL; + hr = IMMDeviceEnumerator_GetDefaultAudioEndpoint (enum_handle, eRender, + eCommunications, &default_render_device); + if (hr == S_OK && default_render_device != NULL) { + hr = IMMDevice_GetId (default_render_device, &default_render_id); + IUnknown_Release(default_render_device); + } + + IMMDevice *default_capture_device = NULL; + wchar_t * default_capture_id = NULL; + hr = IMMDeviceEnumerator_GetDefaultAudioEndpoint (enum_handle, eCapture, + eCommunications, &default_capture_device); + if (hr == S_OK && default_capture_device != NULL) { + hr = IMMDevice_GetId (default_capture_device, &default_capture_id); + IUnknown_Release(default_capture_device); + } + hr = IMMDeviceEnumerator_EnumAudioEndpoints (enum_handle, eAll, dwStateMask, &device_collection); HR_FAILED_GOTO (hr, IMMDeviceEnumerator::EnumAudioEndpoints, err); @@ -433,6 +451,13 @@ gst_wasapi_util_get_devices (GstMMDeviceEnumerator * self, hr = IMMDevice_GetId (item, &wstrid); if (hr != S_OK) goto next; + + gboolean is_default = FALSE; + if (dataflow == eRender && default_render_id && wstrid && wcscmp(default_render_id, wstrid) == 0) + is_default = TRUE; + if (dataflow == eCapture && default_capture_id && wstrid && wcscmp(default_capture_id, wstrid) == 0) + is_default = TRUE; + strid = g_utf16_to_utf8 (wstrid, -1, NULL, NULL, NULL); CoTaskMemFree (wstrid); @@ -477,7 +502,8 @@ gst_wasapi_util_get_devices (GstMMDeviceEnumerator * self, props = gst_structure_new ("wasapi-proplist", "device.api", G_TYPE_STRING, "wasapi", "device.strid", G_TYPE_STRING, GST_STR_NULL (strid), - "wasapi.device.description", G_TYPE_STRING, description, NULL); + "wasapi.device.description", G_TYPE_STRING, description, + "is-default", G_TYPE_BOOLEAN, is_default, NULL); device = g_object_new (GST_TYPE_WASAPI_DEVICE, "device", strid, "display-name", description, "caps", caps, @@ -507,6 +533,10 @@ gst_wasapi_util_get_devices (GstMMDeviceEnumerator * self, res = TRUE; err: + if (default_render_id) + CoTaskMemFree (default_render_id); + if (default_capture_id) + CoTaskMemFree (default_capture_id); if (device_collection) IUnknown_Release (device_collection); return res; From 4b76b613d3b69fab55ecdeed73914ea56f43cae1 Mon Sep 17 00:00:00 2001 From: Knut Saastad Date: Mon, 11 Dec 2023 13:03:06 +0100 Subject: [PATCH 336/339] Revert "avfvideosrc: Don't wait on main thread for permissions request" This reverts commit 855e86f6a287cf362e7fca8bb83f2a9380b99982. --- .../sys/applemedia/avfvideosrc.m | 181 +++++++++--------- 1 file changed, 89 insertions(+), 92 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/avfvideosrc.m b/subprojects/gst-plugins-bad/sys/applemedia/avfvideosrc.m index 19dbd68f3c8..1ceb4065a26 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/avfvideosrc.m +++ b/subprojects/gst-plugins-bad/sys/applemedia/avfvideosrc.m @@ -177,10 +177,6 @@ @interface GstAVFVideoSrcImpl : NSObject change_state (element, transition); + + if (transition == GST_STATE_CHANGE_READY_TO_NULL) + [self closeDevice]; + + return ret; +} + - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)aConnection @@ -1162,6 +1146,8 @@ static void gst_avf_video_src_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_avf_video_src_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); +static GstStateChangeReturn gst_avf_video_src_change_state ( + GstElement * element, GstStateChange transition); static GstCaps * gst_avf_video_src_get_caps (GstBaseSrc * basesrc, GstCaps * filter); static gboolean gst_avf_video_src_set_caps (GstBaseSrc * basesrc, @@ -1193,6 +1179,7 @@ static void gst_avf_video_src_set_context (GstElement * element, gobject_class->get_property = gst_avf_video_src_get_property; gobject_class->set_property = gst_avf_video_src_set_property; + gstelement_class->change_state = gst_avf_video_src_change_state; gstelement_class->set_context = gst_avf_video_src_set_context; gstbasesrc_class->get_caps = gst_avf_video_src_get_caps; @@ -1409,6 +1396,16 @@ static void gst_avf_video_src_set_context (GstElement * element, } } +static GstStateChangeReturn +gst_avf_video_src_change_state (GstElement * element, GstStateChange transition) +{ + GstStateChangeReturn ret; + + ret = [GST_AVF_VIDEO_SRC_IMPL (element) changeState: transition]; + + return ret; +} + static GstCaps * gst_avf_video_src_get_caps (GstBaseSrc * basesrc, GstCaps * filter) { From 823030ed510526fdfe10f4af5a41651af88ffbac Mon Sep 17 00:00:00 2001 From: Knut Saastad Date: Fri, 15 Dec 2023 10:47:16 +0100 Subject: [PATCH 337/339] gstosxaudio: add device is-default info. --- .../sys/osxaudio/gstosxaudiodeviceprovider.c | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c b/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c index 4870fb1a07c..26c93331aeb 100644 --- a/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c +++ b/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c @@ -231,6 +231,41 @@ _audio_system_get_devices (gint * ndevices) return devices; } +static gboolean _audio_system_device_is_default(AudioObjectPropertySelector selector, AudioDeviceID queriedDeviceID) +{ + OSStatus error = noErr; + AudioDeviceID deviceID = 0; + AudioObjectPropertyAddress propertyAddress; + UInt32 propertySize; + + //sets which property to check + propertyAddress.mSelector = selector; + propertyAddress.mScope = kAudioObjectPropertyScopeGlobal; + propertyAddress.mElement = 0; + propertySize = sizeof(AudioDeviceID); + + //gets property (system output device) + error = AudioObjectGetPropertyData( kAudioObjectSystemObject, + &propertyAddress, + 0, + NULL, + &propertySize, + &deviceID); + if (error) + return FALSE; + return deviceID == queriedDeviceID; +} + +static gboolean _audio_system_device_is_default_input(AudioDeviceID queriedDeviceID) +{ + return _audio_system_device_is_default(kAudioHardwarePropertyDefaultInputDevice, queriedDeviceID); +} + +static gboolean _audio_system_device_is_default_output(AudioDeviceID queriedDeviceID) +{ + return _audio_system_device_is_default(kAudioHardwarePropertyDefaultOutputDevice, queriedDeviceID); +} + static void gst_osx_audio_device_provider_probe_internal (GstOsxAudioDeviceProvider * self, gboolean is_src, AudioDeviceID * osx_devices, gint ndevices, @@ -368,6 +403,7 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, g_return_val_if_fail (device_id > 0, NULL); g_return_val_if_fail (device_name, NULL); + gboolean is_default = FALSE; switch (type) { case GST_OSX_AUDIO_DEVICE_TYPE_SOURCE: element_name = "osxaudiosrc"; @@ -376,7 +412,7 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, template_caps = gst_static_pad_template_get_caps (&src_factory); caps = gst_core_audio_probe_caps (core_audio, template_caps); gst_caps_unref (template_caps); - + is_default = _audio_systeM_device_is_default_input(device_id); break; case GST_OSX_AUDIO_DEVICE_TYPE_SINK: element_name = "osxaudiosink"; @@ -385,6 +421,7 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, template_caps = gst_static_pad_template_get_caps (&sink_factory); caps = gst_core_audio_probe_caps (core_audio, template_caps); gst_caps_unref (template_caps); + is_default = _audio_system_device_is_default_output(device_id); break; default: @@ -392,12 +429,14 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, break; } + GstStructure *props = gst_structure_new ("osxaudiodevice-proplist", + "is-default", G_TYPE_BOOLEAN, is_default, NULL); gstdev = g_object_new (GST_TYPE_OSX_AUDIO_DEVICE, "device-id", device_id, "display-name", device_name, "caps", caps, "device-class", - klass, NULL); + klass, "properties", props, NULL); gstdev->element = element_name; - + gst_structure_free (props); return gstdev; } From b00affa908fe296f8f4379ac4462417955e3db78 Mon Sep 17 00:00:00 2001 From: Knut Saastad Date: Fri, 15 Dec 2023 10:52:11 +0100 Subject: [PATCH 338/339] gstosxaudio: fix type in previous commit. --- .../gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c b/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c index 26c93331aeb..3eb0d188d7a 100644 --- a/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c +++ b/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c @@ -412,7 +412,7 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, template_caps = gst_static_pad_template_get_caps (&src_factory); caps = gst_core_audio_probe_caps (core_audio, template_caps); gst_caps_unref (template_caps); - is_default = _audio_systeM_device_is_default_input(device_id); + is_default = _audio_system_device_is_default_input(device_id); break; case GST_OSX_AUDIO_DEVICE_TYPE_SINK: element_name = "osxaudiosink"; From 1ec928e3b6ec52a1cfb35dfedcb539df98efc06b Mon Sep 17 00:00:00 2001 From: Knut Saastad Date: Mon, 18 Dec 2023 14:16:17 +0100 Subject: [PATCH 339/339] gstosxaudiodeviceprovider: add device change monitoring. --- .../sys/osxaudio/gstosxaudiodeviceprovider.c | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c b/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c index 3eb0d188d7a..496b9126e80 100644 --- a/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c +++ b/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c @@ -54,6 +54,14 @@ G_DEFINE_TYPE (GstOsxAudioDeviceProvider, gst_osx_audio_device_provider, static GList *gst_osx_audio_device_provider_probe (GstDeviceProvider * provider); +static gboolean gst_osx_audio_device_provider_start (GstDeviceProvider * provider); +static void gst_osx_audio_device_provider_stop (GstDeviceProvider * provider); +static OSStatus gst_osx_audio_device_change_cb(AudioObjectID inObjectID, + UInt32 inNumberAddresses, + const AudioObjectPropertyAddress* inAddresses, + void* inClientData); +static void +gst_osx_audio_device_provider_update_devices (GstOsxAudioDeviceProvider * provider); static void gst_osx_audio_device_provider_class_init (GstOsxAudioDeviceProviderClass * @@ -62,6 +70,8 @@ gst_osx_audio_device_provider_class_init (GstOsxAudioDeviceProviderClass * GstDeviceProviderClass *dm_class = GST_DEVICE_PROVIDER_CLASS (klass); dm_class->probe = gst_osx_audio_device_provider_probe; + dm_class->start = gst_osx_audio_device_provider_start; + dm_class->stop = gst_osx_audio_device_provider_stop; gst_device_provider_class_set_static_metadata (dm_class, "OSX Audio Device Provider", "Source/Sink/Audio", @@ -69,11 +79,103 @@ gst_osx_audio_device_provider_class_init (GstOsxAudioDeviceProviderClass * "Hyunjun Ko "); } +static OSStatus gst_osx_audio_device_change_cb(AudioObjectID inObjectID, + guint32 inNumberAddresses, + const AudioObjectPropertyAddress* inAddresses, + void* userdata) { + GstOsxAudioDeviceProvider * provider = (GstOsxAudioDeviceProvider *) userdata; + + for (guint32 i = 0; i < inNumberAddresses; i++) { + switch (inAddresses[i].mSelector) { + case kAudioHardwarePropertyDefaultInputDevice: + gst_osx_audio_device_provider_update_devices(provider); + break; + case kAudioHardwarePropertyDefaultOutputDevice: + gst_osx_audio_device_provider_update_devices(provider); + break; + case kAudioHardwarePropertyDevices: + gst_osx_audio_device_provider_update_devices(provider); + break; + default: + break; + } + } + return noErr; +} + static void gst_osx_audio_device_provider_init (GstOsxAudioDeviceProvider * provider) { } +static gboolean +gst_osx_audio_device_provider_start (GstDeviceProvider * provider) +{ + GstOsxAudioDeviceProvider *self = GST_OSX_AUDIO_DEVICE_PROVIDER (provider); + + // Register callbacks for the following AudioObjectIDs + AudioObjectID event_ids[] = {kAudioHardwarePropertyDevices, kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice}; + + for (size_t i = 0; i < sizeof(event_ids) / sizeof(event_ids[0]); i++){ + AudioObjectPropertyAddress deviceListAddr = { + .mSelector = event_ids[i], + .mScope = kAudioObjectPropertyScopeGlobal, + .mElement = kAudioObjectPropertyElementMain + }; + + OSStatus err = + AudioObjectAddPropertyListener(kAudioObjectSystemObject, + &deviceListAddr, + gst_osx_audio_device_change_cb, + (void *)self); + + if (err != noErr) { + GST_ERROR("Failed to register AudioObjectAddPropertyListener(%u) %d", event_ids[i], err); + return FALSE; + } + } + + /* baseclass will not call probe() once it's started, but we can get + * notification only add/remove or change case. To this manually */ + GList *devices = gst_osx_audio_device_provider_probe(provider); + if (devices) { + GList *iter; + for (iter = devices; iter; iter = g_list_next (iter)) { + gst_device_provider_device_add (provider, GST_DEVICE (iter->data)); + } + g_list_free (devices); + } + + return TRUE; +} + +static void +gst_osx_audio_device_provider_stop (GstDeviceProvider * provider) +{ + GstOsxAudioDeviceProvider *self = GST_OSX_AUDIO_DEVICE_PROVIDER (provider); + + // De-register callbacks for the following AudioObjectIDs + AudioObjectID event_ids[] = {kAudioHardwarePropertyDevices, kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice}; + + for (size_t i = 0; i < sizeof(event_ids) / sizeof(event_ids[0]); i++){ + AudioObjectPropertyAddress deviceListAddr = { + .mSelector = event_ids[i], + .mScope = kAudioObjectPropertyScopeGlobal, + .mElement = kAudioObjectPropertyElementMain + }; + + OSStatus err = + AudioObjectRemovePropertyListener(kAudioObjectSystemObject, + &deviceListAddr, + gst_osx_audio_device_change_cb, + (void *)self); + + if (err != noErr) { + GST_ERROR("Failed to de-register AudioObjectAddPropertyListener(%u) %d", event_ids[i], err); + } + } +} + static GstOsxAudioDevice * gst_osx_audio_device_provider_probe_device (GstOsxAudioDeviceProvider * provider, AudioDeviceID device_id, const gchar * device_name, @@ -344,6 +446,99 @@ gst_osx_audio_device_provider_probe (GstDeviceProvider * provider) return devices; } +static gboolean +gst_osx_audio_device_is_in_list (GList * list, GstDevice * device) +{ + GList *iter; + GstStructure *s; + AudioDeviceID device_id; + gboolean device_is_default; + gboolean found = FALSE; + + s = gst_device_get_properties (device); + g_assert (s); + g_assert(gst_structure_get_int (s, "device-id", &device_id) == TRUE); + g_assert(gst_structure_get_boolean (s, "is-default", &device_is_default) == TRUE); + + for (iter = list; iter; iter = g_list_next (iter)) { + GstStructure *other_s; + AudioDeviceID other_device_id; + gboolean other_device_is_default; + + other_s = gst_device_get_properties (GST_DEVICE (iter->data)); + g_assert (other_s); + + g_assert(gst_structure_get_int (other_s, "device-id", &other_device_id) == TRUE); + g_assert(gst_structure_get_boolean (other_s, "is-default", &other_device_is_default) == TRUE); + + if (device_id == other_device_id && device_is_default == other_device_is_default) { + found = TRUE; + } + + gst_structure_free (other_s); + if (found) + break; + } + + gst_structure_free (s); + + return found; +} + +static void +gst_osx_audio_device_provider_update_devices (GstOsxAudioDeviceProvider * self) +{ + GstDeviceProvider *provider = GST_DEVICE_PROVIDER_CAST (self); + GList *prev_devices = NULL; + GList *new_devices = NULL; + GList *to_add = NULL; + GList *to_remove = NULL; + GList *iter; + + GST_OBJECT_LOCK (self); + prev_devices = g_list_copy_deep (provider->devices, + (GCopyFunc) gst_object_ref, NULL); + GST_OBJECT_UNLOCK (self); + + new_devices = gst_osx_audio_device_provider_probe (provider); + + /* Ownership of GstDevice for gst_device_provider_device_add() + * and gst_device_provider_device_remove() is a bit complicated. + * Remove floating reference here for things to be clear */ + for (iter = new_devices; iter; iter = g_list_next (iter)) + gst_object_ref_sink (iter->data); + + /* Check newly added devices */ + for (iter = new_devices; iter; iter = g_list_next (iter)) { + if (!gst_osx_audio_device_is_in_list (prev_devices, GST_DEVICE (iter->data))) { + to_add = g_list_prepend (to_add, gst_object_ref (iter->data)); + } + } + + /* Check removed device */ + for (iter = prev_devices; iter; iter = g_list_next (iter)) { + if (!gst_osx_audio_device_is_in_list (new_devices, GST_DEVICE (iter->data))) { + to_remove = g_list_prepend (to_remove, gst_object_ref (iter->data)); + } + } + + for (iter = to_remove; iter; iter = g_list_next (iter)) + gst_device_provider_device_remove (provider, GST_DEVICE (iter->data)); + + for (iter = to_add; iter; iter = g_list_next (iter)) + gst_device_provider_device_add (provider, GST_DEVICE (iter->data)); + + if (prev_devices) + g_list_free_full (prev_devices, (GDestroyNotify) gst_object_unref); + + if (to_add) + g_list_free_full (to_add, (GDestroyNotify) gst_object_unref); + + if (to_remove) + g_list_free_full (to_remove, (GDestroyNotify) gst_object_unref); +} + + enum { PROP_DEVICE_ID = 1, @@ -430,6 +625,7 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, } GstStructure *props = gst_structure_new ("osxaudiodevice-proplist", + "device-id", G_TYPE_INT, device_id, "is-default", G_TYPE_BOOLEAN, is_default, NULL); gstdev = g_object_new (GST_TYPE_OSX_AUDIO_DEVICE, "device-id", device_id, "display-name", device_name, "caps", caps, "device-class",