TudbuT/projbotv3
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

3.1.0: Add mode for faster internet, use better gif encoder, improve sync fluctuation

Browse source
This commit is contained in:
Daniella / Tove 2023-10-24 22:08:38 +02:00
parent a12f2d1c91
commit b65094dd69
Signed by: TudbuT
GPG key ID: B3CF345217F202D3
5 changed files with 1045 additions and 581 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1458
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

5
Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "projbotv3"
version = "3.0.0"
version = "3.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -11,6 +11,5 @@ serenity = {version="0.11.5", features=["builder", "cache", "client", "framework
songbird = {version="0.3.0", features=["driver", "serenity-rustls"]}
rustls = "0.20"
form-data-builder = "1.0"
png = "0.17"
gif = "0.11"
gifski = "1.12"
webpki-roots = "0.22"

14
README.md
View file

@ -33,3 +33,17 @@ ffmpeg -i vid_30fps.mp4 -vf scale=240:180,setsar=1:1 -deadline realtime vid/%0d.
ffmpeg -i vid.mp4 -deadline realtime aud.opus && mv aud.opus aud_encoded
```
## Assumptions of this implementation
In order to perfectly sync the audio and the video together, some assumptions must be made
which may not always be correct. I'm noting them down here so that future contributors can more
easily debug timing issues.
(non-exhaustive)
- Time it takes to create a voice channel is a good way to measure ping
- It takes about 5 * ping + amount_of_megabytes * ping for the GIF to arrive and display on
the client
- API ping has minimal fluctuation
- Uploading any GIF takes less than 5 seconds
- A GIF of 5 seconds of a reasonable video at 25fps and 320x240 will always fall below 8MB in size

116
src/convert.rs
View file

@ -1,13 +1,21 @@
use std::{
env,
fs::{self, File},
io::Write,
path::PathBuf,
process::{self, Stdio},
sync::{Arc, Mutex},
thread,
time::Duration,
};
use gif::Encoder;
use png::Decoder;
use gifski::{progress::NoProgress, Repeat, Settings};
const MB_1: usize = 1024 * 1024;
const WIDTH: u32 = 320;
const HEIGHT: u32 = 240;
const FAST_INTERNET_SIZE: usize = MB_1 * 3;
pub async fn convert() {
println!("encode: encoding video...");
@ -18,25 +26,7 @@ pub async fn convert() {
"-i",
"vid.mp4",
"-vf",
"fps=fps=25",
"-deadline",
"realtime",
"vid_25fps.mp4",
])
.stdin(Stdio::inherit())
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
.spawn()
.expect("encode: unable to find or run ffmpeg");
command.wait().expect("encode: ffmpeg failed: mp4->mp4");
let mut command = process::Command::new("ffmpeg")
.args([
"-i",
"vid_25fps.mp4",
"-vf",
"scale=240:180,setsar=1:1",
"-deadline",
"realtime",
&format!("scale={WIDTH}:{HEIGHT},setsar=1:1,fps=fps=25"),
"vid/%0d.png",
])
.stdin(Stdio::inherit())
@ -45,9 +35,8 @@ pub async fn convert() {
.spawn()
.expect("encode: unable to find or run ffmpeg");
command.wait().expect("encode: ffmpeg failed: mp4->png");
fs::remove_file("vid_25fps.mp4").expect("encode: rm vid_25fps.mp4 failed");
let mut command = process::Command::new("ffmpeg")
.args(["-i", "vid.mp4", "-deadline", "realtime", "aud.opus"])
.args(["-i", "vid.mp4", "aud.opus"])
.stdin(Stdio::inherit())
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
@ -75,56 +64,45 @@ pub async fn convert() {
thread::spawn(move || {
let mut image = File::create(format!("vid_encoded/{n}"))
.expect("encode: unable to create gif file");
let mut encoder = Some(
Encoder::new(&mut image, 240, 180, &[]).expect("encode: unable to create gif"),
);
// Write the gif control bytes
encoder
.as_mut()
.unwrap()
.write_extension(gif::ExtensionData::new_control_ext(
4,
gif::DisposalMethod::Any,
false,
None,
))
.expect("encode: unable to write extension data");
encoder
.as_mut()
.unwrap()
.set_repeat(gif::Repeat::Finite(0))
.expect("encode: unable to set repeat");
let mut buf = Vec::with_capacity(3 * 1024 * 1024);
let (encoder, writer) = gifski::new(Settings {
width: Some(WIDTH),
height: Some(HEIGHT),
quality: 100,
fast: false,
repeat: Repeat::Finite(0),
})
.expect("unable to start encoder");
thread::spawn(move || {
writer
.write(&mut buf, &mut NoProgress {})
.expect("gif writer failed");
if !env::var("PROJBOTV3_FAST_INTERNET")
.unwrap_or("".to_owned())
.is_empty()
&& buf.len() < FAST_INTERNET_SIZE
{
buf.resize(FAST_INTERNET_SIZE, 0); // extend with zeroes to unify length
}
image
.write_all(buf.as_slice())
.expect("unable to write to file");
*running.lock().unwrap() -= 1;
println!("encode: encoded {n}");
});
// Encode frames into gif
println!("encode: encoding {n}...");
for i in (n * (25 * 5))..dir {
for (gi, i) in ((n * (25 * 5))..dir).enumerate() {
// n number of previously encoded gifs * 25 frames per second * 5 seconds
{
let i = i + 1; // because ffmpeg starts counting at 1 :p
let fi = i + 1; // because ffmpeg starts counting at 1 :p
// Decode frame
let decoder = Decoder::new(
File::open(format!("vid/{i}.png"))
.expect(format!("encode: unable to read vid/{i}.png").as_str()),
);
let mut reader = decoder.read_info().expect(
format!("encode: invalid ffmpeg output in vid/{i}.png").as_str(),
);
let mut buf: Vec<u8> = vec![0; reader.output_buffer_size()];
let info = reader.next_frame(&mut buf).expect(
format!("encode: invalid ffmpeg output in vid/{i}.png").as_str(),
);
let bytes = &mut buf[..info.buffer_size()];
// Encode frame
let mut frame = gif::Frame::from_rgb(240, 180, bytes);
// The gif crate is a little weird with extension data, it writes a
// block for each frame, so we have to remind it of what we want again
// for each frame
frame.delay = 4;
// Add to gif
encoder
.as_mut()
.unwrap()
.write_frame(&frame)
.add_frame_png_file(
gi,
PathBuf::from(format!("vid/{fi}.png")),
gi as f64 / 100.0 * 4.0,
)
.expect("encode: unable to encode frame to gif");
}
// We don't want to encode something that is supposed to go into the next frame
@ -132,15 +110,11 @@ pub async fn convert() {
break;
}
}
*running.lock().unwrap() -= 1;
println!("encode: encoded {n}");
});
}
// Always have 6 running, but no more
while *running.lock().unwrap() >= 6 {
tokio::time::sleep(Duration::from_millis(100)).await;
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
while *running.lock().unwrap() != 0 {
tokio::time::sleep(Duration::from_millis(100)).await;

21
src/main.rs
View file

@ -69,7 +69,6 @@ async fn send_video(message: Message, ctx: Context) {
.expect("discord: unable to send");
// Spawn task to send audio - This has to be here, because this is also where the timer is
// started
tokio::spawn(async move {
let sa = unix_millis();
println!("voice: init");
let channel = guild_id
@ -77,6 +76,7 @@ async fn send_video(message: Message, ctx: Context) {
.await
.expect("voice: unable to create channel");
let api_time = unix_millis() - sa;
tokio::spawn(async move {
*c0.lock().await = Some(channel.id);
println!("voice: joining");
let (handler, err) = songbird.join(guild_id, channel.id).await;
@ -113,8 +113,12 @@ async fn send_video(message: Message, ctx: Context) {
let mut to_compensate_for = 0;
let mut free_time = 0;
const MB_1: usize = 1024 * 1024;
// Send frames (5 second long gifs)
for mut frame in v.by_ref() {
let size_mb = frame.bytes.len() as f32 / MB_1 as f32;
let size_compensation_active = frame.bytes.len() > MB_1;
// Upload the frame to the API, but don't finish off the request.
println!("vid: caching");
let token = token.clone();
@ -204,8 +208,16 @@ async fn send_video(message: Message, ctx: Context) {
}
// Set free_time to display
free_time = to_sleep;
tokio::time::sleep(Duration::from_millis(to_sleep as u64)).await;
sa = unix_millis();
let size_compensation_time = if size_compensation_active {
(api_time as f32 * size_mb) as u64
} else {
0
};
tokio::time::sleep(Duration::from_millis(
to_sleep as u64 - size_compensation_time,
))
.await;
sa = unix_millis() + size_compensation_time;
// Now complete the request. This allows each request to take O(1) time
println!("vid: completing");
@ -214,6 +226,7 @@ async fn send_video(message: Message, ctx: Context) {
})
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(size_compensation_time)).await;
}
// The last frame would immediately be deleted if we didn't wait here.
@ -237,7 +250,7 @@ struct Handler;
#[async_trait]
impl EventHandler for Handler {
async fn message(&self, ctx: Context, message: Message) {
if message.guild_id == None {
if message.guild_id.is_none() {
return;
}