From 00a9424719436c88102a211a311252032848b639 Mon Sep 17 00:00:00 2001 From: Rudi Floren Date: Wed, 6 May 2020 15:36:44 +0200 Subject: [PATCH] feat: sytests --- Cargo.lock | 4 +- Cargo.toml | 4 +- src/client_server.rs | 14 +- src/database.rs | 32 +- src/database/globals.rs | 12 +- src/database/rooms.rs | 10 +- src/main.rs | 3 +- src/server_server.rs | 10 +- sytest/are-we-synapse-yet.list | 836 +++++++++++++++++++++++++++++ sytest/are-we-synapse-yet.py | 260 +++++++++ sytest/show-expected-fail-tests.sh | 105 ++++ sytest/sytest-blacklist | 0 sytest/sytest-whitelist | 85 +++ 13 files changed, 1339 insertions(+), 36 deletions(-) create mode 100644 sytest/are-we-synapse-yet.list create mode 100755 sytest/are-we-synapse-yet.py create mode 100755 sytest/show-expected-fail-tests.sh create mode 100644 sytest/sytest-blacklist create mode 100644 sytest/sytest-whitelist diff --git a/Cargo.lock b/Cargo.lock index 1f1dc949..f059aa40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1216,7 +1216,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.0.1" -source = "git+https://github.com/ruma/ruma-federation-api.git#ccbf216f39bbbaa59131cc200eae5bd18aa1947c" +source = "git+https://github.com/ruma/ruma-federation-api.git?rev=ccbf216f39bbbaa59131cc200eae5bd18aa1947c#ccbf216f39bbbaa59131cc200eae5bd18aa1947c" dependencies = [ "js_int", "ruma-api", @@ -1253,7 +1253,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.6.0-dev.1" -source = "git+https://github.com/ruma/ruma-signatures.git#1ca545cba8dfd43e0fc8e3c18e1311fb73390a97" +source = "git+https://github.com/ruma/ruma-signatures.git?rev=1ca545cba8dfd43e0fc8e3c18e1311fb73390a97#1ca545cba8dfd43e0fc8e3c18e1311fb73390a97" dependencies = [ "base64 0.12.1", "ring", diff --git a/Cargo.toml b/Cargo.toml index 217f58e0..e0a6f10c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ ruma-client-api = "0.8.0" ruma-identifiers = "0.16.1" ruma-api = "0.16.0" ruma-events = "0.21.0" -ruma-signatures = { git = "https://github.com/ruma/ruma-signatures.git" } -ruma-federation-api = { git = "https://github.com/ruma/ruma-federation-api.git" } +ruma-signatures = { git = "https://github.com/ruma/ruma-signatures.git", rev = "1ca545cba8dfd43e0fc8e3c18e1311fb73390a97" } +ruma-federation-api = { git = "https://github.com/ruma/ruma-federation-api.git", rev = "ccbf216f39bbbaa59131cc200eae5bd18aa1947c" } log = "0.4.8" sled = "0.31.0" directories = "2.0.2" diff --git a/src/client_server.rs b/src/client_server.rs index 1a407f79..d32134d3 100644 --- a/src/client_server.rs +++ b/src/client_server.rs @@ -66,7 +66,7 @@ pub fn get_register_available_route( ) -> MatrixResult { // Validate user id let user_id: UserId = - match (*format!("@{}:{}", body.username.clone(), db.globals.hostname())).try_into() { + match (*format!("@{}:{}", body.username.clone(), db.globals.server_name())).try_into() { Err(_) => { debug!("Username invalid"); return MatrixResult(Err(Error { @@ -117,7 +117,7 @@ pub fn register_route( body.username .clone() .unwrap_or_else(|| utils::random_string(GUEST_NAME_LENGTH)), - db.globals.hostname() + db.globals.server_name() )) .try_into() { @@ -229,7 +229,7 @@ pub fn login_route( (body.user.clone(), body.login_info.clone()) { if !username.contains(':') { - username = format!("@{}:{}", username, db.globals.hostname()); + username = format!("@{}:{}", username, db.globals.server_name()); } if let Ok(user_id) = (*username).try_into() { if let Some(hash) = db.users.password_hash(&user_id).unwrap() { @@ -288,7 +288,7 @@ pub fn login_route( MatrixResult(Ok(login::Response { user_id, access_token: token, - home_server: Some(db.globals.hostname().to_owned()), + home_server: Some(db.globals.server_name().to_owned()), device_id, well_known: None, })) @@ -769,7 +769,7 @@ pub fn create_room_route( body: Ruma, ) -> MatrixResult { // TODO: check if room is unique - let room_id = RoomId::try_from(db.globals.hostname()).expect("host is valid"); + let room_id = RoomId::try_from(db.globals.server_name()).expect("host is valid"); let user_id = body.user_id.as_ref().expect("user is authenticated"); db.rooms @@ -858,7 +858,7 @@ pub fn get_alias_route( _room_alias: String, ) -> MatrixResult { // TODO - let room_id = if body.room_alias.server_name() == db.globals.hostname() { + let room_id = if body.room_alias.server_name() == db.globals.server_name() { match body.room_alias.alias() { "conduit" => "!lgOCCXQKtXOAPlAlG5:conduit.rs", _ => { @@ -923,7 +923,7 @@ pub fn join_room_by_id_or_alias_route( let room_id = match RoomId::try_from(body.room_id_or_alias.clone()) { Ok(room_id) => room_id, Err(room_alias) => { - if room_alias.server_name() == db.globals.hostname() { + if room_alias.server_name() == db.globals.server_name() { return MatrixResult(Err(Error { kind: ErrorKind::NotFound, message: "Room alias not found.".to_owned(), diff --git a/src/database.rs b/src/database.rs index 4e6ac57e..0bd3aa02 100644 --- a/src/database.rs +++ b/src/database.rs @@ -7,6 +7,8 @@ pub(self) mod users; use directories::ProjectDirs; use std::fs::remove_dir_all; +use rocket::Config; + pub struct Database { pub globals: globals::Globals, pub users: users::Users, @@ -18,26 +20,38 @@ pub struct Database { impl Database { /// Tries to remove the old database but ignores all errors. - pub fn try_remove(hostname: &str) { + pub fn try_remove(server_name: &str) { let mut path = ProjectDirs::from("xyz", "koesters", "conduit") .unwrap() .data_dir() .to_path_buf(); - path.push(hostname); + path.push(server_name); let _ = remove_dir_all(path); } /// Load an existing database or create a new one. - pub fn load_or_create(hostname: &str) -> Self { - let mut path = ProjectDirs::from("xyz", "koesters", "conduit") - .unwrap() - .data_dir() - .to_path_buf(); - path.push(hostname); + pub fn load_or_create(config: &Config) -> Self { + let server_name = config.get_str("server_name").unwrap_or("localhost"); + + let path = config + .get_str("database_path") + .map(|x| x.to_owned()) + .unwrap_or_else(|_| { + let path = ProjectDirs::from("xyz", "koesters", "conduit") + .unwrap() + .data_dir() + .join(server_name); + path.to_str().unwrap().to_owned() + }); + let db = sled::open(&path).unwrap(); + log::info!("Opened sled database at {}", path); Self { - globals: globals::Globals::load(db.open_tree("global").unwrap(), hostname.to_owned()), + globals: globals::Globals::load( + db.open_tree("global").unwrap(), + server_name.to_owned(), + ), users: users::Users { userid_password: db.open_tree("userid_password").unwrap(), userdeviceids: db.open_tree("userdeviceids").unwrap(), diff --git a/src/database/globals.rs b/src/database/globals.rs index f9e9999b..eb20e373 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -4,13 +4,13 @@ pub const COUNTER: &str = "c"; pub struct Globals { pub(super) globals: sled::Tree, - hostname: String, + server_name: String, keypair: ruma_signatures::Ed25519KeyPair, reqwest_client: reqwest::Client, } impl Globals { - pub fn load(globals: sled::Tree, hostname: String) -> Self { + pub fn load(globals: sled::Tree, server_name: String) -> Self { let keypair = ruma_signatures::Ed25519KeyPair::new( &*globals .update_and_fetch("keypair", utils::generate_keypair) @@ -22,15 +22,15 @@ impl Globals { Self { globals, - hostname, + server_name, keypair, reqwest_client: reqwest::Client::new(), } } - /// Returns the hostname of the server. - pub fn hostname(&self) -> &str { - &self.hostname + /// Returns the server_name of the server. + pub fn server_name(&self) -> &str { + &self.server_name } /// Returns this server's keypair. diff --git a/src/database/rooms.rs b/src/database/rooms.rs index f741afa0..28b35605 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -216,7 +216,7 @@ impl Rooms { event_id: EventId::try_from("$thiswillbefilledinlater").expect("we know this is valid"), room_id: room_id.clone(), sender: sender.clone(), - origin: globals.hostname().to_owned(), + origin: globals.server_name().to_owned(), origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("this only fails many years in the future"), @@ -245,8 +245,12 @@ impl Rooms { .expect("ruma's reference hashes are correct"); let mut pdu_json = serde_json::to_value(&pdu)?; - ruma_signatures::hash_and_sign_event(globals.hostname(), globals.keypair(), &mut pdu_json) - .expect("our new event can be hashed and signed"); + ruma_signatures::hash_and_sign_event( + globals.server_name(), + globals.keypair(), + &mut pdu_json, + ) + .expect("our new event can be hashed and signed"); self.replace_pdu_leaves(&room_id, &pdu.event_id)?; diff --git a/src/main.rs b/src/main.rs index 13b0b654..bef55ac1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,8 +75,7 @@ fn setup_rocket() -> rocket::Rocket { ], ) .attach(AdHoc::on_attach("Config", |rocket| { - let hostname = rocket.config().get_str("hostname").unwrap_or("localhost"); - let data = Database::load_or_create(&hostname); + let data = Database::load_or_create(&rocket.config()); Ok(rocket.manage(data)) })) diff --git a/src/server_server.rs b/src/server_server.rs index f77699e4..2fcbe986 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -55,12 +55,12 @@ pub async fn send_request( request_map.insert("method".to_owned(), T::METADATA.method.to_string().into()); request_map.insert("uri".to_owned(), T::METADATA.path.into()); - request_map.insert("origin".to_owned(), db.globals.hostname().into()); + request_map.insert("origin".to_owned(), db.globals.server_name().into()); request_map.insert("destination".to_owned(), destination.into()); let mut request_json = request_map.into(); ruma_signatures::sign_json( - db.globals.hostname(), + db.globals.server_name(), db.globals.keypair(), &mut request_json, ) @@ -82,7 +82,7 @@ pub async fn send_request( AUTHORIZATION, HeaderValue::from_str(&format!( "X-Matrix origin={},key=\"{}\",sig=\"{}\"", - db.globals.hostname(), + db.globals.server_name(), s.0, s.1 )) @@ -156,7 +156,7 @@ pub fn get_server_keys(db: State<'_, Database>) -> Json { ); let mut response = serde_json::from_slice( http::Response::try_from(get_server_keys::Response { - server_name: db.globals.hostname().to_owned(), + server_name: db.globals.server_name().to_owned(), verify_keys, old_verify_keys: BTreeMap::new(), signatures: BTreeMap::new(), @@ -166,7 +166,7 @@ pub fn get_server_keys(db: State<'_, Database>) -> Json { .body(), ) .unwrap(); - ruma_signatures::sign_json(db.globals.hostname(), db.globals.keypair(), &mut response).unwrap(); + ruma_signatures::sign_json(db.globals.server_name(), db.globals.keypair(), &mut response).unwrap(); Json(response.to_string()) } diff --git a/sytest/are-we-synapse-yet.list b/sytest/are-we-synapse-yet.list new file mode 100644 index 00000000..cdc280a0 --- /dev/null +++ b/sytest/are-we-synapse-yet.list @@ -0,0 +1,836 @@ +reg GET /register yields a set of flows +reg POST /register can create a user +reg POST /register downcases capitals in usernames +reg POST /register returns the same device_id as that in the request +reg POST /register rejects registration of usernames with '!' +reg POST /register rejects registration of usernames with '"' +reg POST /register rejects registration of usernames with ':' +reg POST /register rejects registration of usernames with '?' +reg POST /register rejects registration of usernames with '\' +reg POST /register rejects registration of usernames with '@' +reg POST /register rejects registration of usernames with '[' +reg POST /register rejects registration of usernames with ']' +reg POST /register rejects registration of usernames with '{' +reg POST /register rejects registration of usernames with '|' +reg POST /register rejects registration of usernames with '}' +reg POST /register rejects registration of usernames with '£' +reg POST /register rejects registration of usernames with 'é' +reg POST /register rejects registration of usernames with '\n' +reg POST /register rejects registration of usernames with ''' +reg POST /r0/admin/register with shared secret +reg POST /r0/admin/register admin with shared secret +reg POST /r0/admin/register with shared secret downcases capitals +reg POST /r0/admin/register with shared secret disallows symbols +reg POST rejects invalid utf-8 in JSON +log GET /login yields a set of flows +log POST /login can log in as a user +log POST /login returns the same device_id as that in the request +log POST /login can log in as a user with just the local part of the id +log POST /login as non-existing user is rejected +log POST /login wrong password is rejected +log Interactive authentication types include SSO +log Can perform interactive authentication with SSO +log The user must be consistent through an interactive authentication session with SSO +log The operation must be consistent through an interactive authentication session +v1s GET /events initially +v1s GET /initialSync initially +csa Version responds 200 OK with valid structure +pro PUT /profile/:user_id/displayname sets my name +pro GET /profile/:user_id/displayname publicly accessible +pro PUT /profile/:user_id/avatar_url sets my avatar +pro GET /profile/:user_id/avatar_url publicly accessible +dev GET /device/{deviceId} +dev GET /device/{deviceId} gives a 404 for unknown devices +dev GET /devices +dev PUT /device/{deviceId} updates device fields +dev PUT /device/{deviceId} gives a 404 for unknown devices +dev DELETE /device/{deviceId} +dev DELETE /device/{deviceId} requires UI auth user to match device owner +dev DELETE /device/{deviceId} with no body gives a 401 +dev The deleted device must be consistent through an interactive auth session +dev Users receive device_list updates for their own devices +pre GET /presence/:user_id/status fetches initial status +pre PUT /presence/:user_id/status updates my presence +crm POST /createRoom makes a public room +crm POST /createRoom makes a private room +crm POST /createRoom makes a private room with invites +crm POST /createRoom makes a room with a name +crm POST /createRoom makes a room with a topic +syn Can /sync newly created room +crm POST /createRoom creates a room with the given version +crm POST /createRoom rejects attempts to create rooms with numeric versions +crm POST /createRoom rejects attempts to create rooms with unknown versions +crm POST /createRoom ignores attempts to set the room version via creation_content +mem GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership +mem GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event +rst GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels +mem GET /rooms/:room_id/joined_members fetches my membership +v1s GET /rooms/:room_id/initialSync fetches initial sync state +pub GET /publicRooms lists newly-created room +ali GET /directory/room/:room_alias yields room ID +mem GET /joined_rooms lists newly-created room +rst POST /rooms/:room_id/state/m.room.name sets name +rst GET /rooms/:room_id/state/m.room.name gets name +rst POST /rooms/:room_id/state/m.room.topic sets topic +rst GET /rooms/:room_id/state/m.room.topic gets topic +rst GET /rooms/:room_id/state fetches entire room state +crm POST /createRoom with creation content +ali PUT /directory/room/:room_alias creates alias +nsp GET /rooms/:room_id/aliases lists aliases +jon POST /rooms/:room_id/join can join a room +jon POST /join/:room_alias can join a room +jon POST /join/:room_id can join a room +jon POST /join/:room_id can join a room with custom content +jon POST /join/:room_alias can join a room with custom content +lev POST /rooms/:room_id/leave can leave a room +inv POST /rooms/:room_id/invite can send an invite +ban POST /rooms/:room_id/ban can ban a user +snd POST /rooms/:room_id/send/:event_type sends a message +snd PUT /rooms/:room_id/send/:event_type/:txn_id sends a message +snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id +get GET /rooms/:room_id/messages returns a message +get GET /rooms/:room_id/messages lazy loads members correctly +typ PUT /rooms/:room_id/typing/:user_id sets typing notification +rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels +rst PUT /rooms/:room_id/state/m.room.power_levels can set levels +rst PUT power_levels should not explode if the old power levels were empty +rst Both GET and PUT work +rct POST /rooms/:room_id/receipt can create receipts +red POST /rooms/:room_id/read_markers can create read marker +med POST /media/v1/upload can create an upload +med GET /media/v1/download can fetch the value again +cap GET /capabilities is present and well formed for registered user +cap GET /r0/capabilities is not public +reg Register with a recaptcha +reg registration is idempotent, without username specified +reg registration is idempotent, with username specified +reg registration remembers parameters +reg registration accepts non-ascii passwords +reg registration with inhibit_login inhibits login +reg User signups are forbidden from starting with '_' +reg Can register using an email address +log Can login with 3pid and password using m.login.password +log login types include SSO +log /login/cas/redirect redirects if the old m.login.cas login type is listed +log Can login with new user via CAS +lox Can logout current device +lox Can logout all devices +lox Request to logout with invalid an access token is rejected +lox Request to logout without an access token is rejected +log After changing password, can't log in with old password +log After changing password, can log in with new password +log After changing password, existing session still works +log After changing password, a different session no longer works by default +log After changing password, different sessions can optionally be kept +psh Pushers created with a different access token are deleted on password change +psh Pushers created with a the same access token are not deleted on password change +acc Can deactivate account +acc Can't deactivate account with wrong password +acc After deactivating account, can't log in with password +acc After deactivating account, can't log in with an email +v1s initialSync sees my presence status +pre Presence change reports an event to myself +pre Friends presence changes reports events +crm Room creation reports m.room.create to myself +crm Room creation reports m.room.member to myself +rst Setting room topic reports m.room.topic to myself +v1s Global initialSync +v1s Global initialSync with limit=0 gives no messages +v1s Room initialSync +v1s Room initialSync with limit=0 gives no messages +rst Setting state twice is idempotent +jon Joining room twice is idempotent +syn New room members see their own join event +v1s New room members see existing users' presence in room initialSync +syn Existing members see new members' join events +syn Existing members see new members' presence +v1s All room members see all room members' presence in global initialSync +f,jon Remote users can join room by alias +syn New room members see their own join event +v1s New room members see existing members' presence in room initialSync +syn Existing members see new members' join events +syn Existing members see new member's presence +v1s New room members see first user's profile information in global initialSync +v1s New room members see first user's profile information in per-room initialSync +f,jon Remote users may not join unfederated rooms +syn Local room members see posted message events +v1s Fetching eventstream a second time doesn't yield the message again +syn Local non-members don't see posted message events +get Local room members can get room messages +f,syn Remote room members also see posted message events +f,get Remote room members can get room messages +get Message history can be paginated +f,get Message history can be paginated over federation +eph Ephemeral messages received from clients are correctly expired +ali Room aliases can contain Unicode +f,ali Remote room alias queries can handle Unicode +ali Canonical alias can be set +ali Canonical alias can include alt_aliases +ali Regular users can add and delete aliases in the default room configuration +ali Regular users can add and delete aliases when m.room.aliases is restricted +ali Deleting a non-existent alias should return a 404 +ali Users can't delete other's aliases +ali Users with sufficient power-level can delete other's aliases +ali Can delete canonical alias +ali Alias creators can delete alias with no ops +ali Alias creators can delete canonical alias with no ops +ali Only room members can list aliases of a room +inv Can invite users to invite-only rooms +inv Uninvited users cannot join the room +inv Invited user can reject invite +f,inv Invited user can reject invite over federation +f,inv Invited user can reject invite over federation several times +inv Invited user can reject invite for empty room +f,inv Invited user can reject invite over federation for empty room +inv Invited user can reject local invite after originator leaves +inv Invited user can see room metadata +f,inv Remote invited user can see room metadata +inv Users cannot invite themselves to a room +inv Users cannot invite a user that is already in the room +ban Banned user is kicked and may not rejoin until unbanned +f,ban Remote banned user is kicked and may not rejoin until unbanned +ban 'ban' event respects room powerlevel +plv setting 'm.room.name' respects room powerlevel +plv setting 'm.room.power_levels' respects room powerlevel (2 subtests) +plv Unprivileged users can set m.room.topic if it only needs level 0 +plv Users cannot set ban powerlevel higher than their own (2 subtests) +plv Users cannot set kick powerlevel higher than their own (2 subtests) +plv Users cannot set redact powerlevel higher than their own (2 subtests) +v1s Check that event streams started after a client joined a room work (SYT-1) +v1s Event stream catches up fully after many messages +xxx POST /rooms/:room_id/redact/:event_id as power user redacts message +xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message +xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message +xxx POST /redact disallows redaction of event in different room +xxx Redaction of a redaction redacts the redaction reason +v1s A departed room is still included in /initialSync (SPEC-216) +v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216) +rst Can get rooms/{roomId}/state for a departed room (SPEC-216) +mem Can get rooms/{roomId}/members for a departed room (SPEC-216) +get Can get rooms/{roomId}/messages for a departed room (SPEC-216) +rst Can get 'm.room.name' state for a departed room (SPEC-216) +syn Getting messages going forward is limited for a departed room (SPEC-216) +3pd Can invite existing 3pid +3pd Can invite existing 3pid with no ops into a private room +3pd Can invite existing 3pid in createRoom +3pd Can invite unbound 3pid +f,3pd Can invite unbound 3pid over federation +3pd Can invite unbound 3pid with no ops into a private room +f,3pd Can invite unbound 3pid over federation with no ops into a private room +f,3pd Can invite unbound 3pid over federation with users from both servers +3pd Can accept unbound 3pid invite after inviter leaves +3pd Can accept third party invite with /join +3pd 3pid invite join with wrong but valid signature are rejected +3pd 3pid invite join valid signature but revoked keys are rejected +3pd 3pid invite join valid signature but unreachable ID server are rejected +gst Guest user cannot call /events globally +gst Guest users can join guest_access rooms +gst Guest users can send messages to guest_access rooms if joined +gst Guest user calling /events doesn't tightloop +gst Guest users are kicked from guest_access rooms on revocation of guest_access +gst Guest user can set display names +gst Guest users are kicked from guest_access rooms on revocation of guest_access over federation +gst Guest user can upgrade to fully featured user +gst Guest user cannot upgrade other users +pub GET /publicRooms lists rooms +pub GET /publicRooms includes avatar URLs +gst Guest users can accept invites to private rooms over federation +gst Guest users denied access over federation if guest access prohibited +mem Room members can override their displayname on a room-specific basis +mem Room members can join a room with an overridden displayname +mem Users cannot kick users from a room they are not in +mem Users cannot kick users who have already left a room +typ Typing notification sent to local room members +f,typ Typing notifications also sent to remote room members +typ Typing can be explicitly stopped +rct Read receipts are visible to /initialSync +rct Read receipts are sent as events +rct Receipts must be m.read +pro displayname updates affect room member events +pro avatar_url updates affect room member events +gst m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "shared" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "invited" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "joined" allows/forbids appropriately for Guest users +gst m.room.history_visibility == "default" allows/forbids appropriately for Guest users +gst Guest non-joined user cannot call /events on shared room +gst Guest non-joined user cannot call /events on invited room +gst Guest non-joined user cannot call /events on joined room +gst Guest non-joined user cannot call /events on default room +gst Guest non-joined user can call /events on world_readable room +gst Guest non-joined users can get state for world_readable rooms +gst Guest non-joined users can get individual state for world_readable rooms +gst Guest non-joined users cannot room initalSync for non-world_readable rooms +gst Guest non-joined users can room initialSync for world_readable rooms +gst Guest non-joined users can get individual state for world_readable rooms after leaving +gst Guest non-joined users cannot send messages to guest_access rooms if not joined +gst Guest users can sync from world_readable guest_access rooms if joined +gst Guest users can sync from shared guest_access rooms if joined +gst Guest users can sync from invited guest_access rooms if joined +gst Guest users can sync from joined guest_access rooms if joined +gst Guest users can sync from default guest_access rooms if joined +ath m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users +ath m.room.history_visibility == "shared" allows/forbids appropriately for Real users +ath m.room.history_visibility == "invited" allows/forbids appropriately for Real users +ath m.room.history_visibility == "joined" allows/forbids appropriately for Real users +ath m.room.history_visibility == "default" allows/forbids appropriately for Real users +ath Real non-joined user cannot call /events on shared room +ath Real non-joined user cannot call /events on invited room +ath Real non-joined user cannot call /events on joined room +ath Real non-joined user cannot call /events on default room +ath Real non-joined user can call /events on world_readable room +ath Real non-joined users can get state for world_readable rooms +ath Real non-joined users can get individual state for world_readable rooms +ath Real non-joined users cannot room initalSync for non-world_readable rooms +ath Real non-joined users can room initialSync for world_readable rooms +ath Real non-joined users can get individual state for world_readable rooms after leaving +ath Real non-joined users cannot send messages to guest_access rooms if not joined +ath Real users can sync from world_readable guest_access rooms if joined +ath Real users can sync from shared guest_access rooms if joined +ath Real users can sync from invited guest_access rooms if joined +ath Real users can sync from joined guest_access rooms if joined +ath Real users can sync from default guest_access rooms if joined +ath Only see history_visibility changes on boundaries +f,ath Backfill works correctly with history visibility set to joined +fgt Forgotten room messages cannot be paginated +fgt Forgetting room does not show up in v2 /sync +fgt Can forget room you've been kicked from +fgt Can't forget room you're still in +mem Can re-join room if re-invited +ath Only original members of the room can see messages from erased users +mem /joined_rooms returns only joined rooms +mem /joined_members return joined members +ctx /context/ on joined room works +ctx /context/ on non world readable room does not work +ctx /context/ returns correct number of events +ctx /context/ with lazy_load_members filter works +get /event/ on joined room works +get /event/ on non world readable room does not work +get /event/ does not allow access to events before the user joined +mem Can get rooms/{roomId}/members +mem Can get rooms/{roomId}/members at a given point +mem Can filter rooms/{roomId}/members +upg /upgrade creates a new room +upg /upgrade should preserve room visibility for public rooms +upg /upgrade should preserve room visibility for private rooms +upg /upgrade copies >100 power levels to the new room +upg /upgrade copies the power levels to the new room +upg /upgrade preserves the power level of the upgrading user in old and new rooms +upg /upgrade copies important state to the new room +upg /upgrade copies ban events to the new room +upg local user has push rules copied to upgraded room +f,upg remote user has push rules copied to upgraded room +upg /upgrade moves aliases to the new room +upg /upgrade moves remote aliases to the new room +upg /upgrade preserves direct room state +upg /upgrade preserves room federation ability +upg /upgrade restricts power levels in the old room +upg /upgrade restricts power levels in the old room when the old PLs are unusual +upg /upgrade to an unknown version is rejected +upg /upgrade is rejected if the user can't send state events +upg /upgrade of a bogus room fails gracefully +upg Cannot send tombstone event that points to the same room +f,upg Local and remote users' homeservers remove a room from their public directory on upgrade +rst Name/topic keys are correct +f,pub Can get remote public room list +pub Can paginate public room list +pub Can search public room list +syn Can create filter +syn Can download filter +syn Can sync +syn Can sync a joined room +syn Full state sync includes joined rooms +syn Newly joined room is included in an incremental sync +syn Newly joined room has correct timeline in incremental sync +syn Newly joined room includes presence in incremental sync +syn Get presence for newly joined members in incremental sync +syn Can sync a room with a single message +syn Can sync a room with a message with a transaction id +syn A message sent after an initial sync appears in the timeline of an incremental sync. +syn A filtered timeline reaches its limit +syn Syncing a new room with a large timeline limit isn't limited +syn A full_state incremental update returns only recent timeline +syn A prev_batch token can be used in the v1 messages API +syn A next_batch token can be used in the v1 messages API +syn User sees their own presence in a sync +syn User is offline if they set_presence=offline in their sync +syn User sees updates to presence from other users in the incremental sync. +syn State is included in the timeline in the initial sync +f,syn State from remote users is included in the state in the initial sync +syn Changes to state are included in an incremental sync +syn Changes to state are included in an gapped incremental sync +f,syn State from remote users is included in the timeline in an incremental sync +syn A full_state incremental update returns all state +syn When user joins a room the state is included in the next sync +syn A change to displayname should not result in a full state sync +syn A change to displayname should appear in incremental /sync +syn When user joins a room the state is included in a gapped sync +syn When user joins and leaves a room in the same batch, the full state is still included in the next sync +syn Current state appears in timeline in private history +syn Current state appears in timeline in private history with many messages before +syn Current state appears in timeline in private history with many messages after +syn Rooms a user is invited to appear in an initial sync +syn Rooms a user is invited to appear in an incremental sync +syn Newly joined room is included in an incremental sync after invite +syn Sync can be polled for updates +syn Sync is woken up for leaves +syn Left rooms appear in the leave section of sync +syn Newly left rooms appear in the leave section of incremental sync +syn We should see our own leave event, even if history_visibility is restricted (SYN-662) +syn We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462) +syn Newly left rooms appear in the leave section of gapped sync +syn Previously left rooms don't appear in the leave section of sync +syn Left rooms appear in the leave section of full state sync +syn Archived rooms only contain history from before the user left +syn Banned rooms appear in the leave section of sync +syn Newly banned rooms appear in the leave section of incremental sync +syn Newly banned rooms appear in the leave section of incremental sync +syn Typing events appear in initial sync +syn Typing events appear in incremental sync +syn Typing events appear in gapped sync +syn Read receipts appear in initial v2 /sync +syn New read receipts appear in incremental v2 /sync +syn Can pass a JSON filter as a query parameter +syn Can request federation format via the filter +syn Read markers appear in incremental v2 /sync +syn Read markers appear in initial v2 /sync +syn Read markers can be updated +syn Lazy loading parameters in the filter are strictly boolean +syn The only membership state included in an initial sync is for all the senders in the timeline +syn The only membership state included in an incremental sync is for senders in the timeline +syn The only membership state included in a gapped incremental sync is for senders in the timeline +syn Gapped incremental syncs include all state changes +syn Old leaves are present in gapped incremental syncs +syn Leaves are present in non-gapped incremental syncs +syn Old members are included in gappy incr LL sync if they start speaking +syn Members from the gap are included in gappy incr LL sync +syn We don't send redundant membership state across incremental syncs by default +syn We do send redundant membership state across incremental syncs if asked +syn Unnamed room comes with a name summary +syn Named room comes with just joined member count summary +syn Room summary only has 5 heroes +syn Room summary counts change when membership changes +rmv User can create and send/receive messages in a room with version 1 +rmv User can create and send/receive messages in a room with version 1 (2 subtests) +rmv local user can join room with version 1 +rmv User can invite local user to room with version 1 +rmv remote user can join room with version 1 +rmv User can invite remote user to room with version 1 +rmv Remote user can backfill in a room with version 1 +rmv Can reject invites over federation for rooms with version 1 +rmv Can receive redactions from regular users over federation in room version 1 +rmv User can create and send/receive messages in a room with version 2 +rmv User can create and send/receive messages in a room with version 2 (2 subtests) +rmv local user can join room with version 2 +rmv User can invite local user to room with version 2 +rmv remote user can join room with version 2 +rmv User can invite remote user to room with version 2 +rmv Remote user can backfill in a room with version 2 +rmv Can reject invites over federation for rooms with version 2 +rmv Can receive redactions from regular users over federation in room version 2 +rmv User can create and send/receive messages in a room with version 3 +rmv User can create and send/receive messages in a room with version 3 (2 subtests) +rmv local user can join room with version 3 +rmv User can invite local user to room with version 3 +rmv remote user can join room with version 3 +rmv User can invite remote user to room with version 3 +rmv Remote user can backfill in a room with version 3 +rmv Can reject invites over federation for rooms with version 3 +rmv Can receive redactions from regular users over federation in room version 3 +rmv User can create and send/receive messages in a room with version 4 +rmv User can create and send/receive messages in a room with version 4 (2 subtests) +rmv local user can join room with version 4 +rmv User can invite local user to room with version 4 +rmv remote user can join room with version 4 +rmv User can invite remote user to room with version 4 +rmv Remote user can backfill in a room with version 4 +rmv Can reject invites over federation for rooms with version 4 +rmv Can receive redactions from regular users over federation in room version 4 +rmv User can create and send/receive messages in a room with version 5 +rmv User can create and send/receive messages in a room with version 5 (2 subtests) +rmv local user can join room with version 5 +rmv User can invite local user to room with version 5 +rmv remote user can join room with version 5 +rmv User can invite remote user to room with version 5 +rmv Remote user can backfill in a room with version 5 +rmv Can reject invites over federation for rooms with version 5 +rmv Can receive redactions from regular users over federation in room version 5 +pre Presence changes are reported to local room members +f,pre Presence changes are also reported to remote room members +pre Presence changes to UNAVAILABLE are reported to local room members +f,pre Presence changes to UNAVAILABLE are reported to remote room members +v1s Newly created users see their own presence in /initialSync (SYT-34) +dvk Can upload device keys +dvk Should reject keys claiming to belong to a different user +dvk Can query device keys using POST +dvk Can query specific device keys using POST +dvk query for user with no keys returns empty key dict +dvk Can claim one time key using POST +f,dvk Can query remote device keys using POST +f,dvk Can claim remote one time key using POST +dvk Local device key changes appear in v2 /sync +dvk Local new device changes appear in v2 /sync +dvk Local delete device changes appear in v2 /sync +dvk Local update device changes appear in v2 /sync +dvk Can query remote device keys using POST after notification +f,dev Device deletion propagates over federation +f,dev If remote user leaves room, changes device and rejoins we see update in sync +f,dev If remote user leaves room we no longer receive device updates +dvk Local device key changes appear in /keys/changes +dvk New users appear in /keys/changes +f,dvk If remote user leaves room, changes device and rejoins we see update in /keys/changes +dvk Get left notifs in sync and /keys/changes when other user leaves +dvk Get left notifs for other users in sync and /keys/changes when user leaves +f,dvk If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes +dvk Can create backup version +dvk Can update backup version +dvk Responds correctly when backup is empty +dvk Can backup keys +dvk Can update keys with better versions +dvk Will not update keys with worse versions +dvk Will not back up to an old backup version +dvk Can delete backup +dvk Deleted & recreated backups are empty +dvk Can create more than 10 backup versions +dvk Can upload self-signing keys +dvk Fails to upload self-signing keys with no auth +dvk Fails to upload self-signing key without master key +dvk Changing master key notifies local users +dvk Changing user-signing key notifies local users +f,dvk can fetch self-signing keys over federation +f,dvk uploading self-signing key notifies over federation +f,dvk uploading signed devices gets propagated over federation +tag Can add tag +tag Can remove tag +tag Can list tags for a room +v1s Tags appear in the v1 /events stream +v1s Tags appear in the v1 /initalSync +v1s Tags appear in the v1 room initial sync +tag Tags appear in an initial v2 /sync +tag Newly updated tags appear in an incremental v2 /sync +tag Deleted tags appear in an incremental v2 /sync +tag local user has tags copied to the new room +f,tag remote user has tags copied to the new room +sch Can search for an event by body +sch Can get context around search results +sch Can back-paginate search results +sch Search works across an upgraded room and its predecessor +sch Search results with rank ordering do not include redacted events +sch Search results with recent ordering do not include redacted events +acc Can add account data +acc Can add account data to room +acc Can get account data without syncing +acc Can get room account data without syncing +v1s Latest account data comes down in /initialSync +v1s Latest account data comes down in room initialSync +v1s Account data appears in v1 /events stream +v1s Room account data appears in v1 /events stream +acc Latest account data appears in v2 /sync +acc New account data appears in incremental v2 /sync +oid Can generate a openid access_token that can be exchanged for information about a user +oid Invalid openid access tokens are rejected +oid Requests to userinfo without access tokens are rejected +std Can send a message directly to a device using PUT /sendToDevice +std Can recv a device message using /sync +std Can recv device messages until they are acknowledged +std Device messages with the same txn_id are deduplicated +std Device messages wake up /sync +std Can recv device messages over federation +std Device messages over federation wake up /sync +std Can send messages with a wildcard device id +std Can send messages with a wildcard device id to two devices +std Wildcard device messages wake up /sync +std Wildcard device messages over federation wake up /sync +adm /whois +nsp /purge_history +nsp /purge_history by ts +nsp Can backfill purged history +nsp Shutdown room +ign Ignore user in existing room +ign Ignore invite in full sync +ign Ignore invite in incremental sync +fky Checking local federation server +fky Federation key API allows unsigned requests for keys +fky Federation key API can act as a notary server via a GET request +fky Federation key API can act as a notary server via a POST request +fky Key notary server should return an expired key if it can't find any others +fky Key notary server must not overwrite a valid key with a spurious result from the origin server +fqu Non-numeric ports in server names are rejected +fqu Outbound federation can query profile data +fqu Inbound federation can query profile data +fqu Outbound federation can query room alias directory +fqu Inbound federation can query room alias directory +fsj Outbound federation can query v1 /send_join +fsj Outbound federation can query v2 /send_join +fmj Outbound federation passes make_join failures through to the client +fsj Inbound federation can receive v1 /send_join +fsj Inbound federation can receive v2 /send_join +fmj Inbound /v1/make_join rejects remote attempts to join local users to rooms +fsj Inbound /v1/send_join rejects incorrectly-signed joins +fsj Inbound /v1/send_join rejects joins from other servers +fau Inbound federation rejects remote attempts to kick local users to rooms +frv Inbound federation rejects attempts to join v1 rooms from servers without v1 support +frv Inbound federation rejects attempts to join v2 rooms from servers lacking version support +frv Inbound federation rejects attempts to join v2 rooms from servers only supporting v1 +frv Inbound federation accepts attempts to join v2 rooms from servers with support +frv Outbound federation correctly handles unsupported room versions +frv A pair of servers can establish a join in a v2 room +fsj Outbound federation rejects send_join responses with no m.room.create event +frv Outbound federation rejects m.room.create events with an unknown room version +fsj Event with an invalid signature in the send_join response should not cause room join to fail +fed Outbound federation can send events +fed Inbound federation can receive events +fed Inbound federation can receive redacted events +fed Ephemeral messages received from servers are correctly expired +fed Events whose auth_events are in the wrong room do not mess up the room state +fed Inbound federation can return events +fed Inbound federation redacts events from erased users +fme Outbound federation can request missing events +fme Inbound federation can return missing events for world_readable visibility +fme Inbound federation can return missing events for shared visibility +fme Inbound federation can return missing events for invite visibility +fme Inbound federation can return missing events for joined visibility +fme outliers whose auth_events are in a different room are correctly rejected +fbk Outbound federation can backfill events +fbk Inbound federation can backfill events +fbk Backfill checks the events requested belong to the room +fbk Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination +fiv Outbound federation can send invites via v1 API +fiv Outbound federation can send invites via v2 API +fiv Inbound federation can receive invites via v1 API +fiv Inbound federation can receive invites via v2 API +fiv Inbound federation can receive invite and reject when remote replies with a 403 +fiv Inbound federation can receive invite and reject when remote replies with a 500 +fiv Inbound federation can receive invite and reject when remote is unreachable +fiv Inbound federation rejects invites which are not signed by the sender +fiv Inbound federation can receive invite rejections +fiv Inbound federation rejects incorrectly-signed invite rejections +fsl Inbound /v1/send_leave rejects leaves from other servers +fst Inbound federation can get state for a room +fst Inbound federation of state requires event_id as a mandatory paramater +fst Inbound federation can get state_ids for a room +fst Inbound federation of state_ids requires event_id as a mandatory paramater +fst Federation rejects inbound events where the prev_events cannot be found +fst Room state at a rejected message event is the same as its predecessor +fst Room state at a rejected state event is the same as its predecessor +fst Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state +fst Federation handles empty auth_events in state_ids sanely +fst Getting state checks the events requested belong to the room +fst Getting state IDs checks the events requested belong to the room +fst Should not be able to take over the room by pretending there is no PL event +fpb Inbound federation can get public room list +fed Outbound federation sends receipts +fed Inbound federation rejects receipts from wrong remote +fed Inbound federation ignores redactions from invalid servers room > v3 +fed An event which redacts an event in a different room should be ignored +fed An event which redacts itself should be ignored +fed A pair of events which redact each other should be ignored +fdk Local device key changes get to remote servers +fdk Server correctly handles incoming m.device_list_update +fdk Server correctly resyncs when client query keys and there is no remote cache +fdk Server correctly resyncs when server leaves and rejoins a room +fdk Local device key changes get to remote servers with correct prev_id +fdk Device list doesn't change if remote server is down +fdk If a device list update goes missing, the server resyncs on the next one +fst Name/topic keys are correct +fau Remote servers cannot set power levels in rooms without existing powerlevels +fau Remote servers should reject attempts by non-creators to set the power levels +fau Inbound federation rejects typing notifications from wrong remote +fed Forward extremities remain so even after the next events are populated as outliers +fau Banned servers cannot send events +fau Banned servers cannot /make_join +fau Banned servers cannot /send_join +fau Banned servers cannot /make_leave +fau Banned servers cannot /send_leave +fau Banned servers cannot /invite +fau Banned servers cannot get room state +fau Banned servers cannot get room state ids +fau Banned servers cannot backfill +fau Banned servers cannot /event_auth +fau Banned servers cannot get missing events +fau Server correctly handles transactions that break edu limits +fau Inbound federation correctly soft fails events +fau Inbound federation accepts a second soft-failed event +fau Inbound federation correctly handles soft failed events as extremities +med Can upload with Unicode file name +med Can download with Unicode file name locally +f,med Can download with Unicode file name over federation +med Alternative server names do not cause a routing loop +med Can download specifying a different Unicode file name +med Can upload without a file name +med Can download without a file name locally +f,med Can download without a file name over federation +med Can upload with ASCII file name +med Can download file 'ascii' +med Can download file 'name with spaces' +med Can download file 'name;with;semicolons' +med Can download specifying a different ASCII file name +med Can send image in room message +med Can fetch images in room +med POSTed media can be thumbnailed +f,med Remote media can be thumbnailed +med Test URL preview +med Can read configuration endpoint +nsp Can quarantine media in rooms +udr User appears in user directory +udr User in private room doesn't appear in user directory +udr User joining then leaving public room appears and dissappears from directory +udr Users appear/disappear from directory when join_rules are changed +udr Users appear/disappear from directory when history_visibility are changed +udr Users stay in directory when join_rules are changed but history_visibility is world_readable +f,udr User in remote room doesn't appear in user directory after server left room +udr User directory correctly update on display name change +udr User in shared private room does appear in user directory +udr User in shared private room does appear in user directory until leave +udr User in dir while user still shares private rooms +nsp Create group +nsp Add group rooms +nsp Remove group rooms +nsp Get local group profile +nsp Get local group users +nsp Add/remove local group rooms +nsp Get local group summary +nsp Get remote group profile +nsp Get remote group users +nsp Add/remove remote group rooms +nsp Get remote group summary +nsp Add local group users +nsp Remove self from local group +nsp Remove other from local group +nsp Add remote group users +nsp Remove self from remote group +nsp Listing invited users of a remote group when not a member returns a 403 +nsp Add group category +nsp Remove group category +nsp Get group categories +nsp Add group role +nsp Remove group role +nsp Get group roles +nsp Add room to group summary +nsp Adding room to group summary keeps room_id when fetching rooms in group +nsp Adding multiple rooms to group summary have correct order +nsp Remove room from group summary +nsp Add room to group summary with category +nsp Remove room from group summary with category +nsp Add user to group summary +nsp Adding multiple users to group summary have correct order +nsp Remove user from group summary +nsp Add user to group summary with role +nsp Remove user from group summary with role +nsp Local group invites come down sync +nsp Group creator sees group in sync +nsp Group creator sees group in initial sync +nsp Get/set local group publicity +nsp Bulk get group publicity +nsp Joinability comes down summary +nsp Set group joinable and join it +nsp Group is not joinable by default +nsp Group is joinable over federation +nsp Room is transitioned on local and remote groups upon room upgrade +3pd Can bind 3PID via home server +3pd Can bind and unbind 3PID via homeserver +3pd Can unbind 3PID via homeserver when bound out of band +3pd 3PIDs are unbound after account deactivation +3pd Can bind and unbind 3PID via /unbind by specifying the identity server +3pd Can bind and unbind 3PID via /unbind without specifying the identity server +app AS can create a user +app AS can create a user with an underscore +app AS can create a user with inhibit_login +app AS cannot create users outside its own namespace +app Regular users cannot register within the AS namespace +app AS can make room aliases +app Regular users cannot create room aliases within the AS namespace +app AS-ghosted users can use rooms via AS +app AS-ghosted users can use rooms themselves +app Ghost user must register before joining room +app AS can set avatar for ghosted users +app AS can set displayname for ghosted users +app AS can't set displayname for random users +app Inviting an AS-hosted user asks the AS server +app Accesing an AS-hosted room alias asks the AS server +app Events in rooms with AS-hosted room aliases are sent to AS server +app AS user (not ghost) can join room without registering +app AS user (not ghost) can join room without registering, with user_id query param +app HS provides query metadata +app HS can provide query metadata on a single protocol +app HS will proxy request for 3PU mapping +app HS will proxy request for 3PL mapping +app AS can publish rooms in their own list +app AS and main public room lists are separate +app AS can deactivate a user +psh Test that a message is pushed +psh Invites are pushed +psh Rooms with names are correctly named in pushed +psh Rooms with canonical alias are correctly named in pushed +psh Rooms with many users are correctly pushed +psh Don't get pushed for rooms you've muted +psh Rejected events are not pushed +psh Can add global push rule for room +psh Can add global push rule for sender +psh Can add global push rule for content +psh Can add global push rule for override +psh Can add global push rule for underride +psh Can add global push rule for content +psh New rules appear before old rules by default +psh Can add global push rule before an existing rule +psh Can add global push rule after an existing rule +psh Can delete a push rule +psh Can disable a push rule +psh Adding the same push rule twice is idempotent +psh Messages that notify from another user increment unread notification count +psh Messages that highlight from another user increment unread highlight count +psh Can change the actions of default rules +psh Changing the actions of an unknown default rule fails with 404 +psh Can change the actions of a user specified rule +psh Changing the actions of an unknown rule fails with 404 +psh Can fetch a user's pushers +psh Push rules come down in an initial /sync +psh Adding a push rule wakes up an incremental /sync +psh Disabling a push rule wakes up an incremental /sync +psh Enabling a push rule wakes up an incremental /sync +psh Setting actions for a push rule wakes up an incremental /sync +psh Can enable/disable default rules +psh Enabling an unknown default rule fails with 404 +psh Test that rejected pushers are removed. +psh Notifications can be viewed with GET /notifications +psh Trying to add push rule with no scope fails with 400 +psh Trying to add push rule with invalid scope fails with 400 +psh Trying to add push rule with missing template fails with 400 +psh Trying to add push rule with missing rule_id fails with 400 +psh Trying to add push rule with empty rule_id fails with 400 +psh Trying to add push rule with invalid template fails with 400 +psh Trying to add push rule with rule_id with slashes fails with 400 +psh Trying to add push rule with override rule without conditions fails with 400 +psh Trying to add push rule with underride rule without conditions fails with 400 +psh Trying to add push rule with condition without kind fails with 400 +psh Trying to add push rule with content rule without pattern fails with 400 +psh Trying to add push rule with no actions fails with 400 +psh Trying to add push rule with invalid action fails with 400 +psh Trying to add push rule with invalid attr fails with 400 +psh Trying to add push rule with invalid value for enabled fails with 400 +psh Trying to get push rules with no trailing slash fails with 400 +psh Trying to get push rules with scope without trailing slash fails with 400 +psh Trying to get push rules with template without tailing slash fails with 400 +psh Trying to get push rules with unknown scope fails with 400 +psh Trying to get push rules with unknown template fails with 400 +psh Trying to get push rules with unknown attribute fails with 400 +psh Trying to get push rules with unknown rule_id fails with 404 +v1s GET /initialSync with non-numeric 'limit' +v1s GET /events with non-numeric 'limit' +v1s GET /events with negative 'limit' +v1s GET /events with non-numeric 'timeout' +ath Event size limits +syn Check creating invalid filters returns 4xx +f,pre New federated private chats get full presence information (SYN-115) +pre Left room members do not cause problems for presence +crm Rooms can be created with an initial invite list (SYN-205) +typ Typing notifications don't leak +ban Non-present room members cannot ban others +psh Getting push rules doesn't corrupt the cache SYN-390 +inv Test that we can be reinvited to a room we created +syn Multiple calls to /sync should not cause 500 errors +gst Guest user can call /events on another world_readable room (SYN-606) +gst Real user can call /events on another world_readable room (SYN-606) +gst Events come down the correct room +pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list +std Can send a to-device message to two users which both receive it using /sync diff --git a/sytest/are-we-synapse-yet.py b/sytest/are-we-synapse-yet.py new file mode 100755 index 00000000..0b334ba5 --- /dev/null +++ b/sytest/are-we-synapse-yet.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 + +from __future__ import division +import argparse +import re +import sys + +# Usage: $ ./are-we-synapse-yet.py [-v] results.tap +# This script scans a results.tap file from Dendrite's CI process and spits out +# a rating of how close we are to Synapse parity, based purely on SyTests. +# The main complexity is grouping tests sensibly into features like 'Registration' +# and 'Federation'. Then it just checks the ones which are passing and calculates +# percentages for each group. Produces results like: +# +# Client-Server APIs: 29% (196/666 tests) +# ------------------- +# Registration : 62% (20/32 tests) +# Login : 7% (1/15 tests) +# V1 CS APIs : 10% (3/30 tests) +# ... +# +# or in verbose mode: +# +# Client-Server APIs: 29% (196/666 tests) +# ------------------- +# Registration : 62% (20/32 tests) +# ✓ GET /register yields a set of flows +# ✓ POST /register can create a user +# ✓ POST /register downcases capitals in usernames +# ... +# +# You can also tack `-v` on to see exactly which tests each category falls under. + +test_mappings = { + "nsp": "Non-Spec API", + "f": "Federation", # flag to mark test involves federation + + "federation_apis": { + "fky": "Key API", + "fsj": "send_join API", + "fmj": "make_join API", + "fsl": "send_leave API", + "fiv": "Invite API", + "fqu": "Query API", + "frv": "room versions", + "fau": "Auth", + "fbk": "Backfill API", + "fme": "get_missing_events API", + "fst": "State APIs", + "fpb": "Public Room API", + "fdk": "Device Key APIs", + "fed": "Federation API", + }, + + "client_apis": { + "reg": "Registration", + "log": "Login", + "lox": "Logout", + "v1s": "V1 CS APIs", + "csa": "Misc CS APIs", + "pro": "Profile", + "dev": "Devices", + "dvk": "Device Keys", + "pre": "Presence", + "crm": "Create Room", + "syn": "Sync API", + "rmv": "Room Versions", + "rst": "Room State APIs", + "pub": "Public Room APIs", + "mem": "Room Membership", + "ali": "Room Aliases", + "jon": "Joining Rooms", + "lev": "Leaving Rooms", + "inv": "Inviting users to Rooms", + "ban": "Banning users", + "snd": "Sending events", + "get": "Getting events for Rooms", + "rct": "Receipts", + "red": "Read markers", + "med": "Media APIs", + "cap": "Capabilities API", + "typ": "Typing API", + "psh": "Push APIs", + "acc": "Account APIs", + "eph": "Ephemeral Events", + "plv": "Power Levels", + "xxx": "Redaction", + "3pd": "Third-Party ID APIs", + "gst": "Guest APIs", + "ath": "Room Auth", + "fgt": "Forget APIs", + "ctx": "Context APIs", + "upg": "Room Upgrade APIs", + "tag": "Tagging APIs", + "sch": "Search APIs", + "oid": "OpenID API", + "std": "Send-to-Device APIs", + "adm": "Server Admin API", + "ign": "Ignore Users", + "udr": "User Directory APIs", + "app": "Application Services API", + }, +} + +# optional 'not ' with test number then anything but '#' +re_testname = re.compile(r"^(not )?ok [0-9]+ ([^#]+)") + +# Parses lines like the following: +# +# SUCCESS: ok 3 POST /register downcases capitals in usernames +# FAIL: not ok 54 (expected fail) POST /createRoom creates a room with the given version +# SKIP: ok 821 Multiple calls to /sync should not cause 500 errors # skip lack of can_post_room_receipts +# EXPECT FAIL: not ok 822 (expected fail) Guest user can call /events on another world_readable room (SYN-606) # TODO expected fail +# +# Only SUCCESS lines are treated as success, the rest are not implemented. +# +# Returns a dict like: +# { name: "...", ok: True } +def parse_test_line(line): + if not line.startswith("ok ") and not line.startswith("not ok "): + return + re_match = re_testname.match(line) + test_name = re_match.groups()[1].replace("(expected fail) ", "").strip() + test_pass = False + if line.startswith("ok ") and not "# skip " in line: + test_pass = True + return { + "name": test_name, + "ok": test_pass, + } + +# Prints the stats for a complete section. +# header_name => "Client-Server APIs" +# gid_to_tests => { gid: { : True|False }} +# gid_to_name => { gid: "Group Name" } +# verbose => True|False +# Produces: +# Client-Server APIs: 29% (196/666 tests) +# ------------------- +# Registration : 62% (20/32 tests) +# Login : 7% (1/15 tests) +# V1 CS APIs : 10% (3/30 tests) +# ... +# or in verbose mode: +# Client-Server APIs: 29% (196/666 tests) +# ------------------- +# Registration : 62% (20/32 tests) +# ✓ GET /register yields a set of flows +# ✓ POST /register can create a user +# ✓ POST /register downcases capitals in usernames +# ... +def print_stats(header_name, gid_to_tests, gid_to_name, verbose): + subsections = [] # Registration: 100% (13/13 tests) + subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"] + total_passing = 0 + total_tests = 0 + for gid, tests in gid_to_tests.items(): + group_total = len(tests) + group_passing = 0 + test_names_and_marks = [] + for name, passing in tests.items(): + if passing: + group_passing += 1 + test_names_and_marks.append(f"{'✓' if passing else '×'} {name}") + + total_tests += group_total + total_passing += group_passing + pct = "{0:.0f}%".format(group_passing/group_total * 100) + line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total) + subsections.append(line) + subsection_test_names[line] = test_names_and_marks + + pct = "{0:.0f}%".format(total_passing/total_tests * 100) + print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) + print("-" * (len(header_name)+1)) + for line in subsections: + print(" %s" % (line,)) + if verbose: + for test_name_and_pass_mark in subsection_test_names[line]: + print(" %s" % (test_name_and_pass_mark,)) + print("") + print("") + +def main(results_tap_path, verbose): + # Load up test mappings + test_name_to_group_id = {} + fed_tests = set() + client_tests = set() + groupless_tests = set() + with open("./are-we-synapse-yet.list", "r") as f: + for line in f.readlines(): + test_name = " ".join(line.split(" ")[1:]).strip() + groups = line.split(" ")[0].split(",") + for gid in groups: + if gid == "f" or gid in test_mappings["federation_apis"]: + fed_tests.add(test_name) + else: + client_tests.add(test_name) + if gid == "f": + continue # we expect another group ID + test_name_to_group_id[test_name] = gid + + # parse results.tap + summary = { + "client": { + # gid: { + # test_name: OK + # } + }, + "federation": { + # gid: { + # test_name: OK + # } + }, + "nonspec": { + "nsp": {} + }, + } + with open(results_tap_path, "r") as f: + for line in f.readlines(): + test_result = parse_test_line(line) + if not test_result: + continue + name = test_result["name"] + group_id = test_name_to_group_id.get(name) + if not group_id: + groupless_tests.add(name) + # raise Exception("The test '%s' doesn't have a group" % (name,)) + if group_id == "nsp": + summary["nonspec"]["nsp"][name] = test_result["ok"] + elif group_id in test_mappings["federation_apis"]: + group = summary["federation"].get(group_id, {}) + group[name] = test_result["ok"] + summary["federation"][group_id] = group + elif group_id in test_mappings["client_apis"]: + group = summary["client"].get(group_id, {}) + group[name] = test_result["ok"] + summary["client"][group_id] = group + + print("Are We Synapse Yet?") + print("===================") + print("") + print_stats("Non-Spec APIs", summary["nonspec"], test_mappings, verbose) + print_stats("Client-Server APIs", summary["client"], test_mappings["client_apis"], verbose) + print_stats("Federation APIs", summary["federation"], test_mappings["federation_apis"], verbose) + if verbose: + print("The following tests don't have a group:") + for name in groupless_tests: + print(" %s" % (name,)) + else: + print("%d tests don't have a group" % len(groupless_tests)) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("tap_file", help="path to results.tap") + parser.add_argument("-v", action="store_true", help="show individual test names in output") + args = parser.parse_args() + main(args.tap_file, args.v) diff --git a/sytest/show-expected-fail-tests.sh b/sytest/show-expected-fail-tests.sh new file mode 100755 index 00000000..320d4ebd --- /dev/null +++ b/sytest/show-expected-fail-tests.sh @@ -0,0 +1,105 @@ +#! /bin/bash +# +# Parses a results.tap file from SyTest output and a file containing test names (a test whitelist) +# and checks whether a test name that exists in the whitelist (that should pass), failed or not. +# +# An optional blacklist file can be added, also containing test names, where if a test name is +# present, the script will not error even if the test is in the whitelist file and failed +# +# For each of these files, lines starting with '#' are ignored. +# +# Usage ./show-expected-fail-tests.sh results.tap whitelist [blacklist] + +results_file=$1 +whitelist_file=$2 +blacklist_file=$3 + +fail_build=0 + +if [ $# -lt 2 ]; then + echo "Usage: $0 results.tap whitelist [blacklist]" + exit 1 +fi + +if [ ! -f "$results_file" ]; then + echo "ERROR: Specified results file '${results_file}' doesn't exist." + fail_build=1 +fi + +if [ ! -f "$whitelist_file" ]; then + echo "ERROR: Specified test whitelist '${whitelist_file}' doesn't exist." + fail_build=1 +fi + +blacklisted_tests=() + +# Check if a blacklist file was provided +if [ $# -eq 3 ]; then + # Read test blacklist file + if [ ! -f "$blacklist_file" ]; then + echo "ERROR: Specified test blacklist file '${blacklist_file}' doesn't exist." + fail_build=1 + fi + + # Read each line, ignoring those that start with '#' + blacklisted_tests="" + search_non_comments=$(grep -v '^#' ${blacklist_file}) + while read -r line ; do + # Record the blacklisted test name + blacklisted_tests+=("${line}") + done <<< "${search_non_comments}" # This allows us to edit blacklisted_tests in the while loop +fi + +[ "$fail_build" = 0 ] || exit 1 + +passed_but_expected_fail=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//') +tests_to_add="" +already_in_whitelist="" + +while read -r test_name; do + # Ignore empty lines + [ "${test_name}" = "" ] && continue + + grep "^${test_name}" "${whitelist_file}" > /dev/null 2>&1 + if [ "$?" != "0" ]; then + # Check if this test name is blacklisted + if printf '%s\n' "${blacklisted_tests[@]}" | grep -q -P "^${test_name}$"; then + # Don't notify about this test + continue + fi + + # Append this test_name to the existing list + tests_to_add="${tests_to_add}${test_name}\n" + fail_build=1 + else + already_in_whitelist="${already_in_whitelist}${test_name}\n" + fi +done <<< "${passed_but_expected_fail}" + +# TODO: Check that the same test doesn't exist in both the whitelist and blacklist +# TODO: Check that the same test doesn't appear twice in the whitelist|blacklist + +# Trim test output strings +tests_to_add=$(IFS=$'\n' echo "${tests_to_add[*]%%'\n'}") +already_in_whitelist=$(IFS=$'\n' echo "${already_in_whitelist[*]%%'\n'}") + +# Format output with markdown for buildkite annotation rendering purposes +if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then + echo "### 📜 SyTest Whitelist Maintenance" +fi + +if [ -n "${tests_to_add}" ]; then + echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:" + echo "\`\`\`" + echo -e "${tests_to_add}" + echo "\`\`\`" +fi + +if [ -n "${already_in_whitelist}" ]; then + echo "**WARN**: Tests in the whitelist still marked as **expected fail**:" + echo "\`\`\`" + echo -e "${already_in_whitelist}" + echo "\`\`\`" +fi + +exit ${fail_build} diff --git a/sytest/sytest-blacklist b/sytest/sytest-blacklist new file mode 100644 index 00000000..e69de29b diff --git a/sytest/sytest-whitelist b/sytest/sytest-whitelist new file mode 100644 index 00000000..2c118c27 --- /dev/null +++ b/sytest/sytest-whitelist @@ -0,0 +1,85 @@ +# Register endpoints implemented +GET /register yields a set of flows +POST /register can create a user +POST /register downcases capitals in usernames +POST /register rejects registration of usernames with '!' +POST /register rejects registration of usernames with '"' +POST /register rejects registration of usernames with ':' +POST /register rejects registration of usernames with '?' +POST /register rejects registration of usernames with '\' +POST /register rejects registration of usernames with '@' +POST /register rejects registration of usernames with '[' +POST /register rejects registration of usernames with ']' +POST /register rejects registration of usernames with '{' +POST /register rejects registration of usernames with '|' +POST /register rejects registration of usernames with '}' +POST /register rejects registration of usernames with '£' +POST /register rejects registration of usernames with 'é' +POST /register rejects registration of usernames with '\n' +POST /register rejects registration of usernames with ''' +# Login endpoints implemented +GET /login yields a set of flows +POST /login can log in as a user +POST /login returns the same device_id as that in the request +POST /login can log in as a user with just the local part of the id +POST /login as non-existing user is rejected +POST /login wrong password is rejected +# Room creation endpoints implemented +POST /createRoom makes a public room +POST /createRoom makes a private room +POST /createRoom makes a private room with invites +POST /createRoom makes a room with a name +POST /createRoom makes a room with a topic +Can /sync newly created room +GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership +GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels +POST /join/:room_alias can join a room +POST /join/:room_id can join a room +POST /join/:room_id can join a room with custom content +POST /join/:room_alias can join a room with custom content +POST /rooms/:room_id/join can join a room +POST /rooms/:room_id/leave can leave a room +POST /rooms/:room_id/invite can send an invite +POST /rooms/:room_id/ban can ban a user +POST /rooms/:room_id/send/:event_type sends a message +PUT /rooms/:room_id/send/:event_type/:txn_id sends a message +PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id +GET /rooms/:room_id/state/m.room.power_levels can fetch levels +PUT /rooms/:room_id/state/m.room.power_levels can set levels +PUT power_levels should not explode if the old power levels were empty +Both GET and PUT work +POST /rooms/:room_id/read_markers can create read marker +User signups are forbidden from starting with '_' +Request to logout with invalid an access token is rejected +Request to logout without an access token is rejected +Room creation reports m.room.create to myself +Room creation reports m.room.member to myself +Version responds 200 OK with valid structure +PUT /profile/:user_id/displayname sets my name +GET /profile/:user_id/displayname publicly accessible +GET /device/{deviceId} gives a 404 for unknown devices +PUT /device/{deviceId} gives a 404 for unknown devices +After deactivating account, can't log in with an email +Can create filter +Should reject keys claiming to belong to a different user +Can add account data +Checking local federation server +Alternative server names do not cause a routing loop +Can read configuration endpoint +AS cannot create users outside its own namespace +Changing the actions of an unknown default rule fails with 404 +Changing the actions of an unknown rule fails with 404 +Trying to add push rule with invalid scope fails with 400 +Trying to add push rule with invalid template fails with 400 +Trying to add push rule with rule_id with slashes fails with 400 +Trying to add push rule with override rule without conditions fails with 400 +Trying to add push rule with underride rule without conditions fails with 400 +Trying to add push rule with condition without kind fails with 400 +Trying to add push rule with content rule without pattern fails with 400 +Trying to add push rule with no actions fails with 400 +Trying to add push rule with invalid action fails with 400 +Trying to get push rules with unknown rule_id fails with 404 +GET /events with non-numeric 'limit' +GET /events with negative 'limit' +GET /events with non-numeric 'timeout' +Getting push rules doesn't corrupt the cache SYN-390 \ No newline at end of file