3.1.0: Add mode for faster internet, use better gif encoder, improve sync fluctuation

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "projbotv3"
-version = "3.0.0"
+version = "3.1.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -11,6 +11,5 @@ serenity = {version="0.11.5", features=["builder", "cache", "client", "framework
 songbird = {version="0.3.0", features=["driver", "serenity-rustls"]}
 rustls = "0.20"
 form-data-builder = "1.0"
-png = "0.17"
+gifski = "1.12"
-gif = "0.11"
 webpki-roots = "0.22"

README.md

@@ -33,3 +33,17 @@ ffmpeg -i vid_30fps.mp4 -vf scale=240:180,setsar=1:1 -deadline realtime vid/%0d.
 ffmpeg -i vid.mp4 -deadline realtime aud.opus && mv aud.opus aud_encoded
 ```
 
+## Assumptions of this implementation
+
+In order to perfectly sync the audio and the video together, some assumptions must be made
+which may not always be correct. I'm noting them down here so that future contributors can more
+easily debug timing issues.
+
+(non-exhaustive)
+
+- Time it takes to create a voice channel is a good way to measure ping
+- It takes about 5 * ping + amount_of_megabytes * ping for the GIF to arrive and display on
+  the client
+- API ping has minimal fluctuation
+- Uploading any GIF takes less than 5 seconds
+- A GIF of 5 seconds of a reasonable video at 25fps and 320x240 will always fall below 8MB in size

src/convert.rs

@@ -1,13 +1,21 @@
 use std::{
+    env,
     fs::{self, File},
+    io::Write,
+    path::PathBuf,
     process::{self, Stdio},
     sync::{Arc, Mutex},
     thread,
     time::Duration,
 };
 
-use gif::Encoder;
+use gifski::{progress::NoProgress, Repeat, Settings};
-use png::Decoder;
+
+const MB_1: usize = 1024 * 1024;
+
+const WIDTH: u32 = 320;
+const HEIGHT: u32 = 240;
+const FAST_INTERNET_SIZE: usize = MB_1 * 3;
 
 pub async fn convert() {
     println!("encode: encoding video...");
@@ -18,25 +26,7 @@ pub async fn convert() {
                 "-i",
                 "vid.mp4",
                 "-vf",
-                "fps=fps=25",
+                &format!("scale={WIDTH}:{HEIGHT},setsar=1:1,fps=fps=25"),
-                "-deadline",
-                "realtime",
-                "vid_25fps.mp4",
-            ])
-            .stdin(Stdio::inherit())
-            .stdout(Stdio::inherit())
-            .stderr(Stdio::inherit())
-            .spawn()
-            .expect("encode: unable to find or run ffmpeg");
-        command.wait().expect("encode: ffmpeg failed: mp4->mp4");
-        let mut command = process::Command::new("ffmpeg")
-            .args([
-                "-i",
-                "vid_25fps.mp4",
-                "-vf",
-                "scale=240:180,setsar=1:1",
-                "-deadline",
-                "realtime",
                 "vid/%0d.png",
             ])
             .stdin(Stdio::inherit())
@@ -45,9 +35,8 @@ pub async fn convert() {
             .spawn()
             .expect("encode: unable to find or run ffmpeg");
         command.wait().expect("encode: ffmpeg failed: mp4->png");
-        fs::remove_file("vid_25fps.mp4").expect("encode: rm vid_25fps.mp4 failed");
         let mut command = process::Command::new("ffmpeg")
-            .args(["-i", "vid.mp4", "-deadline", "realtime", "aud.opus"])
+            .args(["-i", "vid.mp4", "aud.opus"])
             .stdin(Stdio::inherit())
             .stdout(Stdio::inherit())
             .stderr(Stdio::inherit())
@@ -75,56 +64,45 @@ pub async fn convert() {
             thread::spawn(move || {
                 let mut image = File::create(format!("vid_encoded/{n}"))
                     .expect("encode: unable to create gif file");
-                let mut encoder = Some(
+                let mut buf = Vec::with_capacity(3 * 1024 * 1024);
-                    Encoder::new(&mut image, 240, 180, &[]).expect("encode: unable to create gif"),
+                let (encoder, writer) = gifski::new(Settings {
-                );
+                    width: Some(WIDTH),
-                // Write the gif control bytes
+                    height: Some(HEIGHT),
-                encoder
+                    quality: 100,
-                    .as_mut()
+                    fast: false,
-                    .unwrap()
+                    repeat: Repeat::Finite(0),
-                    .write_extension(gif::ExtensionData::new_control_ext(
+                })
-                        4,
+                .expect("unable to start encoder");
-                        gif::DisposalMethod::Any,
+                thread::spawn(move || {
-                        false,
+                    writer
-                        None,
+                        .write(&mut buf, &mut NoProgress {})
-                    ))
+                        .expect("gif writer failed");
-                    .expect("encode: unable to write extension data");
+                    if !env::var("PROJBOTV3_FAST_INTERNET")
-                encoder
+                        .unwrap_or("".to_owned())
-                    .as_mut()
+                        .is_empty()
-                    .unwrap()
+                        && buf.len() < FAST_INTERNET_SIZE
-                    .set_repeat(gif::Repeat::Finite(0))
+                    {
-                    .expect("encode: unable to set repeat");
+                        buf.resize(FAST_INTERNET_SIZE, 0); // extend with zeroes to unify length
+                    }
+                    image
+                        .write_all(buf.as_slice())
+                        .expect("unable to write to file");
+                    *running.lock().unwrap() -= 1;
+                    println!("encode: encoded {n}");
+                });
                 // Encode frames into gif
                 println!("encode: encoding {n}...");
-                for i in (n * (25 * 5))..dir {
+                for (gi, i) in ((n * (25 * 5))..dir).enumerate() {
                     // n number of previously encoded gifs * 25 frames per second * 5 seconds
                     {
-                        let i = i + 1; // because ffmpeg starts counting at 1 :p
+                        let fi = i + 1; // because ffmpeg starts counting at 1 :p
 
-                        // Decode frame
-                        let decoder = Decoder::new(
-                            File::open(format!("vid/{i}.png"))
-                                .expect(format!("encode: unable to read vid/{i}.png").as_str()),
-                        );
-                        let mut reader = decoder.read_info().expect(
-                            format!("encode: invalid ffmpeg output in vid/{i}.png").as_str(),
-                        );
-                        let mut buf: Vec<u8> = vec![0; reader.output_buffer_size()];
-                        let info = reader.next_frame(&mut buf).expect(
-                            format!("encode: invalid ffmpeg output in vid/{i}.png").as_str(),
-                        );
-                        let bytes = &mut buf[..info.buffer_size()];
-                        // Encode frame
-                        let mut frame = gif::Frame::from_rgb(240, 180, bytes);
-                        // The gif crate is a little weird with extension data, it writes a
-                        // block for each frame, so we have to remind it of what we want again
-                        // for each frame
-                        frame.delay = 4;
-                        // Add to gif
                         encoder
-                            .as_mut()
+                            .add_frame_png_file(
-                            .unwrap()
+                                gi,
-                            .write_frame(&frame)
+                                PathBuf::from(format!("vid/{fi}.png")),
+                                gi as f64 / 100.0 * 4.0,
+                            )
                             .expect("encode: unable to encode frame to gif");
                     }
                     // We don't want to encode something that is supposed to go into the next frame
@@ -132,15 +110,11 @@ pub async fn convert() {
                         break;
                     }
                 }
-                *running.lock().unwrap() -= 1;
-                println!("encode: encoded {n}");
             });
         }
-        // Always have 6 running, but no more
         while *running.lock().unwrap() >= 6 {
             tokio::time::sleep(Duration::from_millis(100)).await;
         }
-        tokio::time::sleep(Duration::from_millis(500)).await;
     }
     while *running.lock().unwrap() != 0 {
         tokio::time::sleep(Duration::from_millis(100)).await;

src/main.rs

@@ -69,14 +69,14 @@ async fn send_video(message: Message, ctx: Context) {
             .expect("discord: unable to send");
         // Spawn task to send audio - This has to be here, because this is also where the timer is
         // started
+        let sa = unix_millis();
+        println!("voice: init");
+        let channel = guild_id
+            .create_channel(http, |c| c.name("ProjBotV3-Sound").kind(ChannelType::Voice))
+            .await
+            .expect("voice: unable to create channel");
+        let api_time = unix_millis() - sa;
         tokio::spawn(async move {
-            let sa = unix_millis();
-            println!("voice: init");
-            let channel = guild_id
-                .create_channel(http, |c| c.name("ProjBotV3-Sound").kind(ChannelType::Voice))
-                .await
-                .expect("voice: unable to create channel");
-            let api_time = unix_millis() - sa;
             *c0.lock().await = Some(channel.id);
             println!("voice: joining");
             let (handler, err) = songbird.join(guild_id, channel.id).await;
@@ -113,8 +113,12 @@ async fn send_video(message: Message, ctx: Context) {
         let mut to_compensate_for = 0;
         let mut free_time = 0;
 
+        const MB_1: usize = 1024 * 1024;
         // Send frames (5 second long gifs)
         for mut frame in v.by_ref() {
+            let size_mb = frame.bytes.len() as f32 / MB_1 as f32;
+            let size_compensation_active = frame.bytes.len() > MB_1;
+
             // Upload the frame to the API, but don't finish off the request.
             println!("vid: caching");
             let token = token.clone();
@@ -204,8 +208,16 @@ async fn send_video(message: Message, ctx: Context) {
             }
             // Set free_time to display
             free_time = to_sleep;
-            tokio::time::sleep(Duration::from_millis(to_sleep as u64)).await;
+            let size_compensation_time = if size_compensation_active {
-            sa = unix_millis();
+                (api_time as f32 * size_mb) as u64
+            } else {
+                0
+            };
+            tokio::time::sleep(Duration::from_millis(
+                to_sleep as u64 - size_compensation_time,
+            ))
+            .await;
+            sa = unix_millis() + size_compensation_time;
 
             // Now complete the request. This allows each request to take O(1) time
             println!("vid: completing");
@@ -214,6 +226,7 @@ async fn send_video(message: Message, ctx: Context) {
             })
             .await
             .unwrap();
+            tokio::time::sleep(Duration::from_millis(size_compensation_time)).await;
         }
 
         // The last frame would immediately be deleted if we didn't wait here.
@@ -237,7 +250,7 @@ struct Handler;
 #[async_trait]
 impl EventHandler for Handler {
     async fn message(&self, ctx: Context, message: Message) {
-        if message.guild_id == None {
+        if message.guild_id.is_none() {
             return;
         }