3.1.0: Add mode for faster internet, use better gif encoder, improve sync fluctuation
This commit is contained in:
parent
a12f2d1c91
commit
b65094dd69
5 changed files with 1045 additions and 581 deletions
1458
Cargo.lock
generated
1458
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "projbotv3"
|
||||
version = "3.0.0"
|
||||
version = "3.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
@ -11,6 +11,5 @@ serenity = {version="0.11.5", features=["builder", "cache", "client", "framework
|
|||
songbird = {version="0.3.0", features=["driver", "serenity-rustls"]}
|
||||
rustls = "0.20"
|
||||
form-data-builder = "1.0"
|
||||
png = "0.17"
|
||||
gif = "0.11"
|
||||
gifski = "1.12"
|
||||
webpki-roots = "0.22"
|
||||
|
|
14
README.md
14
README.md
|
@ -33,3 +33,17 @@ ffmpeg -i vid_30fps.mp4 -vf scale=240:180,setsar=1:1 -deadline realtime vid/%0d.
|
|||
ffmpeg -i vid.mp4 -deadline realtime aud.opus && mv aud.opus aud_encoded
|
||||
```
|
||||
|
||||
## Assumptions of this implementation
|
||||
|
||||
In order to perfectly sync the audio and the video together, some assumptions must be made
|
||||
which may not always be correct. I'm noting them down here so that future contributors can more
|
||||
easily debug timing issues.
|
||||
|
||||
(non-exhaustive)
|
||||
|
||||
- Time it takes to create a voice channel is a good way to measure ping
|
||||
- It takes about 5 * ping + amount_of_megabytes * ping for the GIF to arrive and display on
|
||||
the client
|
||||
- API ping has minimal fluctuation
|
||||
- Uploading any GIF takes less than 5 seconds
|
||||
- A GIF of 5 seconds of a reasonable video at 25fps and 320x240 will always fall below 8MB in size
|
||||
|
|
116
src/convert.rs
116
src/convert.rs
|
@ -1,13 +1,21 @@
|
|||
use std::{
|
||||
env,
|
||||
fs::{self, File},
|
||||
io::Write,
|
||||
path::PathBuf,
|
||||
process::{self, Stdio},
|
||||
sync::{Arc, Mutex},
|
||||
thread,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use gif::Encoder;
|
||||
use png::Decoder;
|
||||
use gifski::{progress::NoProgress, Repeat, Settings};
|
||||
|
||||
const MB_1: usize = 1024 * 1024;
|
||||
|
||||
const WIDTH: u32 = 320;
|
||||
const HEIGHT: u32 = 240;
|
||||
const FAST_INTERNET_SIZE: usize = MB_1 * 3;
|
||||
|
||||
pub async fn convert() {
|
||||
println!("encode: encoding video...");
|
||||
|
@ -18,25 +26,7 @@ pub async fn convert() {
|
|||
"-i",
|
||||
"vid.mp4",
|
||||
"-vf",
|
||||
"fps=fps=25",
|
||||
"-deadline",
|
||||
"realtime",
|
||||
"vid_25fps.mp4",
|
||||
])
|
||||
.stdin(Stdio::inherit())
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
.spawn()
|
||||
.expect("encode: unable to find or run ffmpeg");
|
||||
command.wait().expect("encode: ffmpeg failed: mp4->mp4");
|
||||
let mut command = process::Command::new("ffmpeg")
|
||||
.args([
|
||||
"-i",
|
||||
"vid_25fps.mp4",
|
||||
"-vf",
|
||||
"scale=240:180,setsar=1:1",
|
||||
"-deadline",
|
||||
"realtime",
|
||||
&format!("scale={WIDTH}:{HEIGHT},setsar=1:1,fps=fps=25"),
|
||||
"vid/%0d.png",
|
||||
])
|
||||
.stdin(Stdio::inherit())
|
||||
|
@ -45,9 +35,8 @@ pub async fn convert() {
|
|||
.spawn()
|
||||
.expect("encode: unable to find or run ffmpeg");
|
||||
command.wait().expect("encode: ffmpeg failed: mp4->png");
|
||||
fs::remove_file("vid_25fps.mp4").expect("encode: rm vid_25fps.mp4 failed");
|
||||
let mut command = process::Command::new("ffmpeg")
|
||||
.args(["-i", "vid.mp4", "-deadline", "realtime", "aud.opus"])
|
||||
.args(["-i", "vid.mp4", "aud.opus"])
|
||||
.stdin(Stdio::inherit())
|
||||
.stdout(Stdio::inherit())
|
||||
.stderr(Stdio::inherit())
|
||||
|
@ -75,56 +64,45 @@ pub async fn convert() {
|
|||
thread::spawn(move || {
|
||||
let mut image = File::create(format!("vid_encoded/{n}"))
|
||||
.expect("encode: unable to create gif file");
|
||||
let mut encoder = Some(
|
||||
Encoder::new(&mut image, 240, 180, &[]).expect("encode: unable to create gif"),
|
||||
);
|
||||
// Write the gif control bytes
|
||||
encoder
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.write_extension(gif::ExtensionData::new_control_ext(
|
||||
4,
|
||||
gif::DisposalMethod::Any,
|
||||
false,
|
||||
None,
|
||||
))
|
||||
.expect("encode: unable to write extension data");
|
||||
encoder
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.set_repeat(gif::Repeat::Finite(0))
|
||||
.expect("encode: unable to set repeat");
|
||||
let mut buf = Vec::with_capacity(3 * 1024 * 1024);
|
||||
let (encoder, writer) = gifski::new(Settings {
|
||||
width: Some(WIDTH),
|
||||
height: Some(HEIGHT),
|
||||
quality: 100,
|
||||
fast: false,
|
||||
repeat: Repeat::Finite(0),
|
||||
})
|
||||
.expect("unable to start encoder");
|
||||
thread::spawn(move || {
|
||||
writer
|
||||
.write(&mut buf, &mut NoProgress {})
|
||||
.expect("gif writer failed");
|
||||
if !env::var("PROJBOTV3_FAST_INTERNET")
|
||||
.unwrap_or("".to_owned())
|
||||
.is_empty()
|
||||
&& buf.len() < FAST_INTERNET_SIZE
|
||||
{
|
||||
buf.resize(FAST_INTERNET_SIZE, 0); // extend with zeroes to unify length
|
||||
}
|
||||
image
|
||||
.write_all(buf.as_slice())
|
||||
.expect("unable to write to file");
|
||||
*running.lock().unwrap() -= 1;
|
||||
println!("encode: encoded {n}");
|
||||
});
|
||||
// Encode frames into gif
|
||||
println!("encode: encoding {n}...");
|
||||
for i in (n * (25 * 5))..dir {
|
||||
for (gi, i) in ((n * (25 * 5))..dir).enumerate() {
|
||||
// n number of previously encoded gifs * 25 frames per second * 5 seconds
|
||||
{
|
||||
let i = i + 1; // because ffmpeg starts counting at 1 :p
|
||||
let fi = i + 1; // because ffmpeg starts counting at 1 :p
|
||||
|
||||
// Decode frame
|
||||
let decoder = Decoder::new(
|
||||
File::open(format!("vid/{i}.png"))
|
||||
.expect(format!("encode: unable to read vid/{i}.png").as_str()),
|
||||
);
|
||||
let mut reader = decoder.read_info().expect(
|
||||
format!("encode: invalid ffmpeg output in vid/{i}.png").as_str(),
|
||||
);
|
||||
let mut buf: Vec<u8> = vec![0; reader.output_buffer_size()];
|
||||
let info = reader.next_frame(&mut buf).expect(
|
||||
format!("encode: invalid ffmpeg output in vid/{i}.png").as_str(),
|
||||
);
|
||||
let bytes = &mut buf[..info.buffer_size()];
|
||||
// Encode frame
|
||||
let mut frame = gif::Frame::from_rgb(240, 180, bytes);
|
||||
// The gif crate is a little weird with extension data, it writes a
|
||||
// block for each frame, so we have to remind it of what we want again
|
||||
// for each frame
|
||||
frame.delay = 4;
|
||||
// Add to gif
|
||||
encoder
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.write_frame(&frame)
|
||||
.add_frame_png_file(
|
||||
gi,
|
||||
PathBuf::from(format!("vid/{fi}.png")),
|
||||
gi as f64 / 100.0 * 4.0,
|
||||
)
|
||||
.expect("encode: unable to encode frame to gif");
|
||||
}
|
||||
// We don't want to encode something that is supposed to go into the next frame
|
||||
|
@ -132,15 +110,11 @@ pub async fn convert() {
|
|||
break;
|
||||
}
|
||||
}
|
||||
*running.lock().unwrap() -= 1;
|
||||
println!("encode: encoded {n}");
|
||||
});
|
||||
}
|
||||
// Always have 6 running, but no more
|
||||
while *running.lock().unwrap() >= 6 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
while *running.lock().unwrap() != 0 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
|
33
src/main.rs
33
src/main.rs
|
@ -69,14 +69,14 @@ async fn send_video(message: Message, ctx: Context) {
|
|||
.expect("discord: unable to send");
|
||||
// Spawn task to send audio - This has to be here, because this is also where the timer is
|
||||
// started
|
||||
let sa = unix_millis();
|
||||
println!("voice: init");
|
||||
let channel = guild_id
|
||||
.create_channel(http, |c| c.name("ProjBotV3-Sound").kind(ChannelType::Voice))
|
||||
.await
|
||||
.expect("voice: unable to create channel");
|
||||
let api_time = unix_millis() - sa;
|
||||
tokio::spawn(async move {
|
||||
let sa = unix_millis();
|
||||
println!("voice: init");
|
||||
let channel = guild_id
|
||||
.create_channel(http, |c| c.name("ProjBotV3-Sound").kind(ChannelType::Voice))
|
||||
.await
|
||||
.expect("voice: unable to create channel");
|
||||
let api_time = unix_millis() - sa;
|
||||
*c0.lock().await = Some(channel.id);
|
||||
println!("voice: joining");
|
||||
let (handler, err) = songbird.join(guild_id, channel.id).await;
|
||||
|
@ -113,8 +113,12 @@ async fn send_video(message: Message, ctx: Context) {
|
|||
let mut to_compensate_for = 0;
|
||||
let mut free_time = 0;
|
||||
|
||||
const MB_1: usize = 1024 * 1024;
|
||||
// Send frames (5 second long gifs)
|
||||
for mut frame in v.by_ref() {
|
||||
let size_mb = frame.bytes.len() as f32 / MB_1 as f32;
|
||||
let size_compensation_active = frame.bytes.len() > MB_1;
|
||||
|
||||
// Upload the frame to the API, but don't finish off the request.
|
||||
println!("vid: caching");
|
||||
let token = token.clone();
|
||||
|
@ -204,8 +208,16 @@ async fn send_video(message: Message, ctx: Context) {
|
|||
}
|
||||
// Set free_time to display
|
||||
free_time = to_sleep;
|
||||
tokio::time::sleep(Duration::from_millis(to_sleep as u64)).await;
|
||||
sa = unix_millis();
|
||||
let size_compensation_time = if size_compensation_active {
|
||||
(api_time as f32 * size_mb) as u64
|
||||
} else {
|
||||
0
|
||||
};
|
||||
tokio::time::sleep(Duration::from_millis(
|
||||
to_sleep as u64 - size_compensation_time,
|
||||
))
|
||||
.await;
|
||||
sa = unix_millis() + size_compensation_time;
|
||||
|
||||
// Now complete the request. This allows each request to take O(1) time
|
||||
println!("vid: completing");
|
||||
|
@ -214,6 +226,7 @@ async fn send_video(message: Message, ctx: Context) {
|
|||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(size_compensation_time)).await;
|
||||
}
|
||||
|
||||
// The last frame would immediately be deleted if we didn't wait here.
|
||||
|
@ -237,7 +250,7 @@ struct Handler;
|
|||
#[async_trait]
|
||||
impl EventHandler for Handler {
|
||||
async fn message(&self, ctx: Context, message: Message) {
|
||||
if message.guild_id == None {
|
||||
if message.guild_id.is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue