Reduce latency by 1 frame

rom1v · rom1v · commit 63af7fbafe37 · 2019-07-31T01:55:32.000+02:00
To packetize the H.264 raw stream, av_parser_parse2() (called by av_read_frame()) knows that it has received a full frame only after it has received some data for the next frame. As a consequence, the client always waited until the next frame before sending the current frame to the decoder! On the device side, we know packets boundaries. To reduce latency, make the device always transmit the "frame meta" to packetize the stream manually (it was already implemented to send PTS, but only enabled on recording). On the client side, replace av_read_frame() by manual packetizing and parsing. <https://stackoverflow.com/questions/50682518/replacing-av-read-frame-to-reduce-delay> <https://trac.ffmpeg.org/ticket/3354>
diff --git a/app/src/recorder.c b/app/src/recorder.c
@@ -166,11 +166,21 @@ recorder_rescale_packet(struct recorder *recorder, AVPacket *packet) {
 bool
 recorder_write(struct recorder *recorder, AVPacket *packet) {
     if (!recorder->header_written) {
+        if (packet->pts != AV_NOPTS_VALUE) {
+            LOGE("The first packet is not a config packet");
+            return false;
+        }
         bool ok = recorder_write_header(recorder, packet);
         if (!ok) {
             return false;
         }
         recorder->header_written = true;
+        return true;
+    }
+
+    if (packet->pts == AV_NOPTS_VALUE) {
+        // ignore config packets
+        return true;
     }
 
     recorder_rescale_packet(recorder, packet);
diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c
@@ -277,7 +277,6 @@ scrcpy(const struct scrcpy_options *options) {
         .local_port = options->port,
         .max_size = options->max_size,
         .bit_rate = options->bit_rate,
-        .send_frame_meta = record,
         .control = options->control,
     };
     if (!server_start(&server, options->serial, &params)) {
diff --git a/app/src/server.c b/app/src/server.c
@@ -130,7 +130,7 @@ execute_server(struct server *server, const struct server_params *params) {
         bit_rate_string,
         server->tunnel_forward ? "true" : "false",
         params->crop ? params->crop : "-",
-        params->send_frame_meta ? "true" : "false",
+        "true", // always send frame meta (packet boundaries + timestamp)
         params->control ? "true" : "false",
     };
     return adb_execute(server->serial, cmd, sizeof(cmd) / sizeof(cmd[0]));
diff --git a/app/src/server.h b/app/src/server.h
@@ -34,7 +34,6 @@ struct server_params {
     uint16_t local_port;
     uint16_t max_size;
     uint32_t bit_rate;
-    bool send_frame_meta;
     bool control;
 };
 
diff --git a/app/src/stream.c b/app/src/stream.c
diff --git a/app/src/stream.h b/app/src/stream.h
