nero/tsuki
1
0
Fork 0
Code Issues 2 Pull requests Projects Releases Packages Wiki Activity Actions

feat: implement appservices

Browse source 
this also reverts some stateres changes
This commit is contained in:
Timo Kösters 2020-12-08 10:33:44 +01:00
parent d62f17a91a
commit 6e5b35ea92
No known key found for this signature in database
GPG key ID: 24DA7517711A2BA4
26 changed files with 696 additions and 584 deletions
Download patch file Download diff file Expand all files Collapse all files

104
src/appservice_server.rs Normal file
View file

@ -0,0 +1,104 @@
use crate::{utils, Error, Result};
use http::header::{HeaderValue, CONTENT_TYPE};
use log::warn;
use ruma::api::OutgoingRequest;
use std::{
convert::{TryFrom, TryInto},
fmt::Debug,
time::Duration,
};
pub async fn send_request<T: OutgoingRequest>(
globals: &crate::database::globals::Globals,
registration: serde_yaml::Value,
request: T,
) -> Result<T::IncomingResponse>
where
T: Debug,
{
let destination = registration.get("url").unwrap().as_str().unwrap();
let hs_token = registration.get("hs_token").unwrap().as_str().unwrap();
let mut http_request = request
.try_into_http_request(&destination, Some(""))
.unwrap();
let mut parts = http_request.uri().clone().into_parts();
let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned();
let symbol = if old_path_and_query.contains("?") {
"&"
} else {
"?"
};
parts.path_and_query = Some(
(old_path_and_query + symbol + "access_token=" + hs_token)
.parse()
.unwrap(),
);
*http_request.uri_mut() = parts.try_into().expect("our manipulation is always valid");
http_request.headers_mut().insert(
CONTENT_TYPE,
HeaderValue::from_str("application/json").unwrap(),
);
let mut reqwest_request = reqwest::Request::try_from(http_request)
.expect("all http requests are valid reqwest requests");
*reqwest_request.timeout_mut() = Some(Duration::from_secs(30));
let url = reqwest_request.url().clone();
let reqwest_response = globals.reqwest_client().execute(reqwest_request).await;
// Because reqwest::Response -> http::Response is complicated:
match reqwest_response {
Ok(mut reqwest_response) => {
let status = reqwest_response.status();
let mut http_response = http::Response::builder().status(status);
let headers = http_response.headers_mut().unwrap();
for (k, v) in reqwest_response.headers_mut().drain() {
if let Some(key) = k {
headers.insert(key, v);
}
}
let status = reqwest_response.status();
let body = reqwest_response
.bytes()
.await
.unwrap_or_else(|e| {
warn!("server error: {}", e);
Vec::new().into()
}) // TODO: handle timeout
.into_iter()
.collect::<Vec<_>>();
if status != 200 {
warn!(
"Server returned bad response {} ({}): {} {:?}",
destination,
url,
status,
utils::string_from_bytes(&body)
);
}
let response = T::IncomingResponse::try_from(
http_response
.body(body)
.expect("reqwest body is valid http body"),
);
response.map_err(|_| {
warn!(
"Server returned invalid response bytes {} ({})",
destination, url
);
Error::BadServerResponse("Server returned bad response.")
})
}
Err(e) => Err(e.into()),
}
}

35
src/client_server/account.rs
View file

@ -139,18 +139,20 @@ pub async fn register_route(
auth_error: None,
};
if let Some(auth) = &body.auth {
let (worked, uiaainfo) =
db.uiaa
.try_auth(&user_id, "".into(), auth, &uiaainfo, &db.users, &db.globals)?;
if !worked {
if !body.from_appservice {
if let Some(auth) = &body.auth {
let (worked, uiaainfo) =
db.uiaa
.try_auth(&user_id, "".into(), auth, &uiaainfo, &db.users, &db.globals)?;
if !worked {
return Err(Error::Uiaa(uiaainfo));
}
// Success!
} else {
uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
db.uiaa.create(&user_id, "".into(), &uiaainfo)?;
return Err(Error::Uiaa(uiaainfo));
}
// Success!
} else {
uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
db.uiaa.create(&user_id, "".into(), &uiaainfo)?;
return Err(Error::Uiaa(uiaainfo));
}
if missing_username {
@ -241,6 +243,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 2. Make conduit bot join
@ -265,6 +268,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 3. Power levels
@ -302,6 +306,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 4.1 Join Rules
@ -322,6 +327,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 4.2 History Visibility
@ -344,6 +350,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 4.3 Guest Access
@ -364,6 +371,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 6. Events implied by name and topic
@ -386,6 +394,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.rooms.build_and_append_pdu(
@ -405,6 +414,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// Room alias
@ -430,6 +440,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?;
@ -456,6 +467,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.rooms.build_and_append_pdu(
PduBuilder {
@ -478,6 +490,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// Send welcome message
@ -506,6 +519,7 @@ pub async fn register_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
}
@ -681,6 +695,7 @@ pub async fn deactivate_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
}

41
src/client_server/alias.rs
View file

@ -1,7 +1,8 @@
use super::State;
use crate::{server_server, ConduitResult, Database, Error, Ruma};
use crate::{appservice_server, server_server, ConduitResult, Database, Error, Ruma};
use ruma::{
api::{
appservice,
client::{
error::ErrorKind,
r0::alias::{create_alias, delete_alias, get_alias},
@ -75,13 +76,37 @@ pub async fn get_alias_helper(
return Ok(get_alias::Response::new(response.room_id, response.servers).into());
}
let room_id = db
.rooms
.id_from_alias(&room_alias)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"Room with alias not found.",
))?;
let mut room_id = None;
match db.rooms.id_from_alias(&room_alias)? {
Some(r) => room_id = Some(r),
None => {
for (_id, registration) in db.appservice.iter_all().filter_map(|r| r.ok()) {
if appservice_server::send_request(
&db.globals,
registration,
appservice::query::query_room_alias::v1::Request { room_alias },
)
.await
.is_ok()
{
room_id = Some(db.rooms.id_from_alias(&room_alias)?.ok_or_else(|| {
Error::bad_config("Appservice lied to us. Room does not exist.")
})?);
break;
}
}
}
};
let room_id = match room_id {
Some(room_id) => room_id,
None => {
return Err(Error::BadRequest(
ErrorKind::NotFound,
"Room with alias not found.",
))
}
};
Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_owned()]).into())
}

2
src/client_server/media.rs
View file

@ -45,7 +45,7 @@ pub async fn create_content_route(
db.flush().await?;
Ok(create_content::Response { content_uri: mxc }.into())
Ok(create_content::Response { content_uri: mxc, blurhash: None }.into())
}
#[cfg_attr(

6
src/client_server/membership.rs
View file

@ -128,6 +128,7 @@ pub async fn leave_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.flush().await?;
@ -167,6 +168,7 @@ pub async fn invite_user_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.flush().await?;
@ -222,6 +224,7 @@ pub async fn kick_user_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.flush().await?;
@ -281,6 +284,7 @@ pub async fn ban_user_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.flush().await?;
@ -332,6 +336,7 @@ pub async fn unban_user_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.flush().await?;
@ -713,6 +718,7 @@ async fn join_room_by_id_helper(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
}

3
src/client_server/message.rs
View file

@ -22,7 +22,7 @@ pub async fn send_message_event_route(
body: Ruma<send_message_event::Request<'_>>,
) -> ConduitResult<send_message_event::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_deref();
// Check if this is a new transaction id
if let Some(response) =
@ -69,6 +69,7 @@ pub async fn send_message_event_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.transaction_ids.add_txnid(

2
src/client_server/profile.rs
View file

@ -67,6 +67,7 @@ pub async fn set_displayname_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// Presence update
@ -163,6 +164,7 @@ pub async fn set_avatar_url_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// Presence update

1
src/client_server/redact.rs
View file

@ -35,6 +35,7 @@ pub async fn redact_event_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.flush().await?;

15
src/client_server/room.rs
View file

@ -69,6 +69,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 2. Let the room creator join
@ -93,6 +94,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 3. Power levels
@ -137,6 +139,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 4. Events set by preset
@ -176,6 +179,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 4.2 History Visibility
@ -196,6 +200,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 4.3 Guest Access
@ -224,6 +229,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// 5. Events listed in initial_state
@ -246,6 +252,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
}
@ -270,6 +277,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
}
@ -291,6 +299,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
}
@ -317,6 +326,7 @@ pub async fn create_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
}
@ -407,6 +417,7 @@ pub async fn upgrade_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// Get the old room federations status
@ -450,6 +461,7 @@ pub async fn upgrade_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// Join the new room
@ -474,6 +486,7 @@ pub async fn upgrade_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
// Recommended transferable state events list from the specs
@ -510,6 +523,7 @@ pub async fn upgrade_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
}
@ -556,6 +570,7 @@ pub async fn upgrade_room_route(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
db.flush().await?;

3
src/client_server/state.rs
View file

@ -63,8 +63,8 @@ pub async fn send_state_event_for_empty_key_route(
let Ruma {
body,
sender_user,
sender_device: _,
json_body,
..
} = body;
let json = serde_json::from_str::<serde_json::Value>(
@ -288,6 +288,7 @@ pub async fn send_state_event_for_key_helper(
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)?;
Ok(event_id)

2
src/client_server/to_device.rs
View file

@ -17,7 +17,7 @@ pub async fn send_event_to_device_route(
body: Ruma<send_event_to_device::Request<'_>>,
) -> ConduitResult<send_event_to_device::Response> {
let sender_user = body.sender_user.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_ref().expect("user is authenticated");
let sender_device = body.sender_device.as_deref();
// Check if this is a new transaction id
if db

8
src/database.rs
View file

@ -1,5 +1,6 @@
pub mod account_data;
pub mod admin;
pub mod appservice;
pub mod globals;
pub mod key_backups;
pub mod media;
@ -16,6 +17,8 @@ use log::info;
use rocket::futures::{self, channel::mpsc};
use ruma::{DeviceId, ServerName, UserId};
use serde::Deserialize;
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::{convert::TryInto, fs::remove_dir_all};
#[derive(Clone, Deserialize)]
@ -59,6 +62,7 @@ pub struct Database {
pub transaction_ids: transaction_ids::TransactionIds,
pub sending: sending::Sending,
pub admin: admin::Admin,
pub appservice: appservice::Appservice,
pub _db: sled::Db,
}
@ -180,6 +184,10 @@ impl Database {
admin: admin::Admin {
sender: admin_sender,
},
appservice: appservice::Appservice {
cached_registrations: Arc::new(RwLock::new(HashMap::new())),
id_appserviceregistrations: db.open_tree("id_appserviceregistrations")?,
},
_db: db,
};

61
src/database/admin.rs
View file

@ -10,7 +10,9 @@ use ruma::{
use tokio::select;
pub enum AdminCommand {
SendTextMessage(message::TextMessageEventContent),
RegisterAppservice(serde_yaml::Value),
ListAppservices,
SendMessage(message::MessageEventContent),
}
#[derive(Clone)]
@ -44,28 +46,49 @@ impl Admin {
warn!("Conduit instance does not have an #admins room. Logging to that room will not work.");
}
let send_message = |message: message::MessageEventContent| {
if let Some(conduit_room) = &conduit_room {
db.rooms
.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMessage,
content: serde_json::to_value(message)
.expect("event is valid, we just created it"),
unsigned: None,
state_key: None,
redacts: None,
},
&conduit_user,
&conduit_room,
&db.globals,
&db.sending,
&db.admin,
&db.account_data,
&db.appservice,
)
.unwrap();
}
};
loop {
select! {
Some(event) = receiver.next() => {
match event {
AdminCommand::SendTextMessage(message) => {
if let Some(conduit_room) = &conduit_room {
db.rooms.build_and_append_pdu(
PduBuilder {
event_type: EventType::RoomMessage,
content: serde_json::to_value(message).expect("event is valid, we just created it"),
unsigned: None,
state_key: None,
redacts: None,
},
&conduit_user,
&conduit_room,
&db.globals,
&db.sending,
&db.admin,
&db.account_data,
).unwrap();
}
AdminCommand::RegisterAppservice(yaml) => {
db.appservice.register_appservice(yaml).unwrap(); // TODO handle error
}
AdminCommand::ListAppservices => {
let appservices = db.appservice.iter_ids().collect::<Vec<_>>();
let count = appservices.len();
let output = format!(
"Appservices ({}): {}",
count,
appservices.into_iter().filter_map(|r| r.ok()).collect::<Vec<_>>().join(", ")
);
send_message(message::MessageEventContent::text_plain(output));
}
AdminCommand::SendMessage(message) => {
send_message(message);
}
}
}

67
src/database/appservice.rs Normal file
View file

@ -0,0 +1,67 @@
use crate::{utils, Error, Result};
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
#[derive(Clone)]
pub struct Appservice {
pub(super) cached_registrations: Arc<RwLock<HashMap<String, serde_yaml::Value>>>,
pub(super) id_appserviceregistrations: sled::Tree,
}
impl Appservice {
pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> {
// TODO: Rumaify
let id = yaml.get("id").unwrap().as_str().unwrap();
self.id_appserviceregistrations
.insert(id, serde_yaml::to_string(&yaml).unwrap().as_bytes())?;
self.cached_registrations
.write()
.unwrap()
.insert(id.to_owned(), yaml);
Ok(())
}
pub fn get_registration(&self, id: &str) -> Result<Option<serde_yaml::Value>> {
self.cached_registrations
.read()
.unwrap()
.get(id)
.map_or_else(
|| {
Ok(self
.id_appserviceregistrations
.get(id)?
.map(|bytes| {
Ok::<_, Error>(serde_yaml::from_slice(&bytes).map_err(|_| {
Error::bad_database(
"Invalid registration bytes in id_appserviceregistrations.",
)
})?)
})
.transpose()?)
},
|r| Ok(Some(r.clone())),
)
}
pub fn iter_ids(&self) -> impl Iterator<Item = Result<String>> {
self.id_appserviceregistrations.iter().keys().map(|id| {
Ok(utils::string_from_bytes(&id?).map_err(|_| {
Error::bad_database("Invalid id bytes in id_appserviceregistrations.")
})?)
})
}
pub fn iter_all<'a>(
&'a self,
) -> impl Iterator<Item = Result<(String, serde_yaml::Value)>> + 'a {
self.iter_ids().filter_map(|id| id.ok()).map(move |id| {
Ok((
id.clone(),
self.get_registration(&id)?
.expect("iter_ids only returns appservices that exist"),
))
})
}
}

12
src/database/globals.rs
View file

@ -1,10 +1,10 @@
use crate::{database::Config, utils, Error, Result};
use trust_dns_resolver::TokioAsyncResolver;
use std::collections::HashMap;
use log::error;
use ruma::ServerName;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::RwLock;
use trust_dns_resolver::TokioAsyncResolver;
pub const COUNTER: &str = "c";
@ -59,9 +59,11 @@ impl Globals {
config,
keypair: Arc::new(keypair),
reqwest_client: reqwest::Client::new(),
dns_resolver: TokioAsyncResolver::tokio_from_system_conf().await.map_err(|_| {
Error::bad_config("Failed to set up trust dns resolver with system config.")
})?,
dns_resolver: TokioAsyncResolver::tokio_from_system_conf()
.await
.map_err(|_| {
Error::bad_config("Failed to set up trust dns resolver with system config.")
})?,
actual_destination_cache: Arc::new(RwLock::new(HashMap::new())),
})
}

7
src/database/media.rs
View file

@ -290,7 +290,12 @@ impl Media {
file: thumbnail_bytes.to_vec(),
}))
} else {
Ok(None)
// Couldn't parse file to generate thumbnail, send original
Ok(Some(FileMeta {
filename,
content_type,
file: file.to_vec(),
}))
}
} else {
Ok(None)

122
src/database/rooms.rs
View file

@ -36,16 +36,6 @@ use super::admin::AdminCommand;
/// hashing the entire state.
pub type StateHashId = IVec;
/// An enum that represents the two valid states when searching
/// for an events "parent".
///
/// An events parent is any event we are aware of that is part of
/// the events `prev_events` array.
pub(crate) enum ClosestParent {
Append,
Insert(u64),
}
#[derive(Clone)]
pub struct Rooms {
pub edus: edus::RoomEdus,
@ -411,54 +401,6 @@ impl Rooms {
}
}
/// Recursively search for a PDU from our DB that is also in the
/// `prev_events` field of the incoming PDU.
///
/// First we check if the last PDU inserted to the given room is a parent
/// if not we recursively check older `prev_events` to insert the incoming
/// event after.
pub(crate) fn get_latest_pduid_before(
&self,
room: &RoomId,
incoming_prev_ids: &[EventId],
their_state: &BTreeMap<EventId, Arc<StateEvent>>,
) -> Result<Option<ClosestParent>> {
match self.pduid_pdu.scan_prefix(room.as_bytes()).last() {
Some(Ok(val))
if incoming_prev_ids.contains(
&serde_json::from_slice::<PduEvent>(&val.1)
.map_err(|_| {
Error::bad_database("last DB entry contains invalid PDU bytes")
})?
.event_id,
) =>
{
Ok(Some(ClosestParent::Append))
}
_ => {
let mut prev_ids = incoming_prev_ids.to_vec();
while let Some(id) = prev_ids.pop() {
match self.get_pdu_id(&id)? {
Some(pdu_id) => {
return Ok(Some(ClosestParent::Insert(self.pdu_count(&pdu_id)?)));
}
None => {
prev_ids.extend(their_state.get(&id).map_or(
Err(Error::BadServerResponse(
"Failed to find previous event for PDU in state",
)),
// `prev_event_ids` will return an empty Vec instead of failing
// so it works perfect for our use here
|pdu| Ok(pdu.prev_event_ids()),
)?);
}
}
}
Ok(None)
}
}
}
/// Returns the leaf pdus of a room.
pub fn get_pdu_leaves(&self, room_id: &RoomId) -> Result<Vec<EventId>> {
let mut prefix = room_id.as_bytes().to_vec();
@ -583,18 +525,59 @@ impl Rooms {
.as_ref()
== Some(&pdu.room_id)
{
let mut parts = body.split_whitespace().skip(1);
let mut lines = body.lines();
let command_line = lines.next().expect("each string has at least one line");
let body = lines.collect::<Vec<_>>();
let mut parts = command_line.split_whitespace().skip(1);
if let Some(command) = parts.next() {
let args = parts.collect::<Vec<_>>();
admin.send(AdminCommand::SendTextMessage(
message::TextMessageEventContent {
body: format!("Command: {}, Args: {:?}", command, args),
formatted: None,
relates_to: None,
new_content: None,
},
));
match command {
"register_appservice" => {
if body.len() > 2
&& body[0].trim() == "```"
&& body.last().unwrap().trim() == "```"
{
let appservice_config = body[1..body.len() - 1].join("\n");
let parsed_config = serde_yaml::from_str::<serde_yaml::Value>(
&appservice_config,
);
match parsed_config {
Ok(yaml) => {
admin.send(AdminCommand::RegisterAppservice(yaml));
}
Err(e) => {
admin.send(AdminCommand::SendMessage(
message::MessageEventContent::text_plain(
format!(
"Could not parse appservice config: {}",
e
),
),
));
}
}
} else {
admin.send(AdminCommand::SendMessage(
message::MessageEventContent::text_plain(
"Expected code block in command body.",
),
));
}
}
"list_appservices" => {
admin.send(AdminCommand::ListAppservices);
}
_ => {
admin.send(AdminCommand::SendMessage(
message::MessageEventContent::text_plain(format!(
"Command: {}, Args: {:?}",
command, args
)),
));
}
}
}
}
}
@ -675,6 +658,7 @@ impl Rooms {
sending: &super::sending::Sending,
admin: &super::admin::Admin,
account_data: &super::account_data::AccountData,
appservice: &super::appservice::Appservice,
) -> Result<EventId> {
let PduBuilder {
event_type,
@ -923,6 +907,10 @@ impl Rooms {
sending.send_pdu(&server, &pdu_id)?;
}
for appservice in appservice.iter_all().filter_map(|r| r.ok()) {
sending.send_pdu_appservice(&appservice.0, &pdu_id)?;
}
Ok(pdu.event_id)
}

240
src/database/sending.rs
View file

@ -1,26 +1,35 @@
use std::{collections::HashMap, convert::TryFrom, time::SystemTime};
use crate::{server_server, utils, Error, PduEvent, Result};
use crate::{appservice_server, server_server, utils, Error, PduEvent, Result};
use federation::transactions::send_transaction_message;
use log::{debug, warn};
use log::warn;
use rocket::futures::stream::{FuturesUnordered, StreamExt};
use ruma::{api::federation, ServerName};
use ruma::{
api::{appservice, federation},
ServerName,
};
use sled::IVec;
use tokio::select;
#[derive(Clone)]
pub struct Sending {
/// The state for a given state hash.
pub(super) servernamepduids: sled::Tree, // ServernamePduId = ServerName + PduId
pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = ServerName + PduId (pduid can be empty for reservation)
pub(super) servernamepduids: sled::Tree, // ServernamePduId = (+)ServerName + PduId
pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = (+)ServerName + PduId (pduid can be empty for reservation)
}
impl Sending {
pub fn start_handler(&self, globals: &super::globals::Globals, rooms: &super::rooms::Rooms) {
pub fn start_handler(
&self,
globals: &super::globals::Globals,
rooms: &super::rooms::Rooms,
appservice: &super::appservice::Appservice,
) {
let servernamepduids = self.servernamepduids.clone();
let servercurrentpdus = self.servercurrentpdus.clone();
let rooms = rooms.clone();
let globals = globals.clone();
let appservice = appservice.clone();
tokio::spawn(async move {
let mut futures = FuturesUnordered::new();
@ -28,7 +37,7 @@ impl Sending {
// Retry requests we could not finish yet
let mut current_transactions = HashMap::new();
for (server, pdu) in servercurrentpdus
for (server, pdu, is_appservice) in servercurrentpdus
.iter()
.filter_map(|r| r.ok())
.map(|(key, _)| {
@ -38,45 +47,61 @@ impl Sending {
Error::bad_database("Invalid bytes in servercurrentpdus.")
})?;
let server = utils::string_from_bytes(&server).map_err(|_| {
Error::bad_database("Invalid server bytes in server_currenttransaction")
})?;
// Appservices start with a plus
let (server, is_appservice) = if server.starts_with("+") {
(&server[1..], true)
} else {
(&*server, false)
};
Ok::<_, Error>((
Box::<ServerName>::try_from(utils::string_from_bytes(&server).map_err(
|_| {
Error::bad_database(
"Invalid server bytes in server_currenttransaction",
)
},
)?)
.map_err(|_| {
Box::<ServerName>::try_from(server).map_err(|_| {
Error::bad_database(
"Invalid server string in server_currenttransaction",
)
})?,
IVec::from(pdu),
is_appservice,
))
})
.filter_map(|r| r.ok())
.filter(|(_, pdu)| !pdu.is_empty()) // Skip reservation key
.filter(|(_, pdu, _)| !pdu.is_empty()) // Skip reservation key
.take(50)
// This should not contain more than 50 anyway
{
current_transactions
.entry(server)
.entry((server, is_appservice))
.or_insert_with(Vec::new)
.push(pdu);
}
for (server, pdus) in current_transactions {
futures.push(Self::handle_event(server, pdus, &globals, &rooms));
for ((server, is_appservice), pdus) in current_transactions {
futures.push(Self::handle_event(
server,
is_appservice,
pdus,
&globals,
&rooms,
&appservice,
));
}
let mut subscriber = servernamepduids.watch_prefix(b"");
loop {
select! {
Some(server) = futures.next() => {
debug!("sending response: {:?}", &server);
match server {
Ok((server, _response)) => {
let mut prefix = server.as_bytes().to_vec();
Some(response) = futures.next() => {
match response {
Ok((server, is_appservice)) => {
let mut prefix = if is_appservice {
"+".as_bytes().to_vec()
} else {
Vec::new()
};
prefix.extend_from_slice(server.as_bytes());
prefix.push(0xff);
for key in servercurrentpdus
@ -109,13 +134,13 @@ impl Sending {
servernamepduids.remove(&current_key).unwrap();
}
futures.push(Self::handle_event(server, new_pdus, &globals, &rooms));
futures.push(Self::handle_event(server, is_appservice, new_pdus, &globals, &rooms, &appservice));
} else {
servercurrentpdus.remove(&prefix).unwrap();
// servercurrentpdus with the prefix should be empty now
}
}
Err((server, e)) => {
Err((server, _is_appservice, e)) => {
warn!("Couldn't send transaction to {}: {}", server, e)
// TODO: exponential backoff
}
@ -126,24 +151,37 @@ impl Sending {
let servernamepduid = key.clone();
let mut parts = servernamepduid.splitn(2, |&b| b == 0xff);
if let Some((server, pdu_id)) = utils::string_from_bytes(
if let Some((server, is_appservice, pdu_id)) = utils::string_from_bytes(
parts
.next()
.expect("splitn will always return 1 or more elements"),
)
.map_err(|_| Error::bad_database("ServerName in servernamepduid bytes are invalid."))
.and_then(|server_str| Box::<ServerName>::try_from(server_str)
.map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid.")))
.map(|server_str| {
// Appservices start with a plus
if server_str.starts_with("+") {
(server_str[1..].to_owned(), true)
} else {
(server_str, false)
}
})
.and_then(|(server_str, is_appservice)| Box::<ServerName>::try_from(server_str)
.map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid.")).map(|s| (s, is_appservice)))
.ok()
.and_then(|server| parts
.and_then(|(server, is_appservice)| parts
.next()
.ok_or_else(|| Error::bad_database("Invalid servernamepduid in db."))
.ok()
.map(|pdu_id| (server, pdu_id))
.map(|pdu_id| (server, is_appservice, pdu_id))
)
// TODO: exponential backoff
.filter(|(server, _)| {
let mut prefix = server.to_string().as_bytes().to_vec();
.filter(|(server, is_appservice, _)| {
let mut prefix = if *is_appservice {
"+".as_bytes().to_vec()
} else {
Vec::new()
};
prefix.extend_from_slice(server.as_bytes());
prefix.push(0xff);
servercurrentpdus
@ -154,7 +192,7 @@ impl Sending {
servercurrentpdus.insert(&key, &[]).unwrap();
servernamepduids.remove(&key).unwrap();
futures.push(Self::handle_event(server, vec![pdu_id.into()], &globals, &rooms));
futures.push(Self::handle_event(server, is_appservice, vec![pdu_id.into()], &globals, &rooms, &appservice));
}
}
}
@ -172,56 +210,102 @@ impl Sending {
Ok(())
}
pub fn send_pdu_appservice(&self, appservice_id: &str, pdu_id: &[u8]) -> Result<()> {
let mut key = "+".as_bytes().to_vec();
key.extend_from_slice(appservice_id.as_bytes());
key.push(0xff);
key.extend_from_slice(pdu_id);
self.servernamepduids.insert(key, b"")?;
Ok(())
}
async fn handle_event(
server: Box<ServerName>,
is_appservice: bool,
pdu_ids: Vec<IVec>,
globals: &super::globals::Globals,
rooms: &super::rooms::Rooms,
) -> std::result::Result<
(Box<ServerName>, send_transaction_message::v1::Response),
(Box<ServerName>, Error),
> {
let pdu_jsons = pdu_ids
.iter()
.map(|pdu_id| {
Ok::<_, (Box<ServerName>, Error)>(
// TODO: check room version and remove event_id if needed
serde_json::from_str(
PduEvent::convert_to_outgoing_federation_event(
rooms
.get_pdu_json_from_id(pdu_id)
.map_err(|e| (server.clone(), e))?
.ok_or_else(|| {
(
server.clone(),
Error::bad_database(
"Event in servernamepduids not found in db.",
),
)
})?,
)
.json()
.get(),
appservice: &super::appservice::Appservice,
) -> std::result::Result<(Box<ServerName>, bool), (Box<ServerName>, bool, Error)> {
if is_appservice {
let pdu_jsons = pdu_ids
.iter()
.map(|pdu_id| {
Ok::<_, (Box<ServerName>, Error)>(
rooms
.get_pdu_from_id(pdu_id)
.map_err(|e| (server.clone(), e))?
.ok_or_else(|| {
(
server.clone(),
Error::bad_database(
"Event in servernamepduids not found in db.",
),
)
})?
.to_any_event(),
)
.expect("Raw<..> is always valid"),
)
})
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
})
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
appservice_server::send_request(
&globals,
appservice
.get_registration(server.as_str())
.unwrap()
.unwrap(), // TODO: handle error
appservice::event::push_events::v1::Request {
events: &pdu_jsons,
txn_id: &utils::random_string(16),
},
)
.await
.map(|_response| (server.clone(), is_appservice))
.map_err(|e| (server, is_appservice, e))
} else {
let pdu_jsons = pdu_ids
.iter()
.map(|pdu_id| {
Ok::<_, (Box<ServerName>, Error)>(
// TODO: check room version and remove event_id if needed
serde_json::from_str(
PduEvent::convert_to_outgoing_federation_event(
rooms
.get_pdu_json_from_id(pdu_id)
.map_err(|e| (server.clone(), e))?
.ok_or_else(|| {
(
server.clone(),
Error::bad_database(
"Event in servernamepduids not found in db.",
),
)
})?,
)
.json()
.get(),
)
.expect("Raw<..> is always valid"),
)
})
.filter_map(|r| r.ok())
.collect::<Vec<_>>();
server_server::send_request(
&globals,
server.clone(),
send_transaction_message::v1::Request {
origin: globals.server_name(),
pdus: &pdu_jsons,
edus: &[],
origin_server_ts: SystemTime::now(),
transaction_id: &utils::random_string(16),
},
)
.await
.map(|response| (server.clone(), response))
.map_err(|e| (server, e))
server_server::send_request(
&globals,
server.clone(),
send_transaction_message::v1::Request {
origin: globals.server_name(),
pdus: &pdu_jsons,
edus: &[],
origin_server_ts: SystemTime::now(),
transaction_id: &utils::random_string(16),
},
)
.await
.map(|_response| (server.clone(), is_appservice))
.map_err(|e| (server, is_appservice, e))
}
}
}

8
src/database/transaction_ids.rs
View file

@ -11,13 +11,13 @@ impl TransactionIds {
pub fn add_txnid(
&self,
user_id: &UserId,
device_id: &DeviceId,
device_id: Option<&DeviceId>,
txn_id: &str,
data: &[u8],
) -> Result<()> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(device_id.as_bytes());
key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default());
key.push(0xff);
key.extend_from_slice(txn_id.as_bytes());
@ -29,12 +29,12 @@ impl TransactionIds {
pub fn existing_txnid(
&self,
user_id: &UserId,
device_id: &DeviceId,
device_id: Option<&DeviceId>,
txn_id: &str,
) -> Result<Option<IVec>> {
let mut key = user_id.as_bytes().to_vec();
key.push(0xff);
key.extend_from_slice(device_id.as_bytes());
key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default());
key.push(0xff);
key.extend_from_slice(txn_id.as_bytes());

4
src/error.rs
View file

@ -142,8 +142,8 @@ impl log::Log for ConduitLogger {
mut_last_logs.insert(output.clone(), Instant::now());
}
self.db.admin.send(AdminCommand::SendTextMessage(
message::TextMessageEventContent::plain(output),
self.db.admin.send(AdminCommand::SendMessage(
message::MessageEventContent::text_plain(output),
));
}
}

1
src/lib.rs
View file

@ -1,3 +1,4 @@
pub mod appservice_server;
pub mod client_server;
mod database;
mod error;

4
src/main.rs
View file

@ -1,5 +1,6 @@
#![warn(rust_2018_idioms)]
pub mod appservice_server;
pub mod client_server;
pub mod server_server;
@ -139,7 +140,8 @@ fn setup_rocket() -> rocket::Rocket {
.await
.expect("config is valid");
data.sending.start_handler(&data.globals, &data.rooms);
data.sending
.start_handler(&data.globals, &data.rooms, &data.appservice);
log::set_boxed_logger(Box::new(ConduitLogger {
db: data.clone(),
last_logs: Default::default(),

88
src/ruma_wrapper.rs
View file

@ -12,7 +12,7 @@ use std::{
#[cfg(feature = "conduit_bin")]
use {
crate::utils,
log::warn,
log::{debug, warn},
rocket::{
data::{
ByteUnit, Data, FromDataFuture, FromTransformedData, Transform, TransformFuture,
@ -34,6 +34,7 @@ pub struct Ruma<T: Outgoing> {
pub sender_user: Option<UserId>,
pub sender_device: Option<Box<DeviceId>>,
pub json_body: Option<Box<serde_json::value::RawValue>>, // This is None when body is not a valid string
pub from_appservice: bool,
}
#[cfg(feature = "conduit_bin")]
@ -66,28 +67,72 @@ where
.await
.expect("database was loaded");
let (sender_user, sender_device) = match T::METADATA.authentication {
AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
// Get token from header or query value
let token = match request
.headers()
.get_one("Authorization")
.map(|s| s[7..].to_owned()) // Split off "Bearer "
.or_else(|| request.get_query_value("access_token").and_then(|r| r.ok()))
{
// TODO: M_MISSING_TOKEN
None => return Failure((Status::Unauthorized, ())),
Some(token) => token,
};
// Get token from header or query value
let token = request
.headers()
.get_one("Authorization")
.map(|s| s[7..].to_owned()) // Split off "Bearer "
.or_else(|| request.get_query_value("access_token").and_then(|r| r.ok()));
// Check if token is valid
match db.users.find_from_token(&token).unwrap() {
// TODO: M_UNKNOWN_TOKEN
None => return Failure((Status::Unauthorized, ())),
Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())),
let (sender_user, sender_device, from_appservice) = if let Some((_id, registration)) =
db.appservice
.iter_all()
.filter_map(|r| r.ok())
.find(|(_id, registration)| {
registration
.get("as_token")
.and_then(|as_token| as_token.as_str())
.map_or(false, |as_token| token.as_deref() == Some(as_token))
}) {
match T::METADATA.authentication {
AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
let user_id = request.get_query_value::<String>("user_id").map_or_else(
|| {
UserId::parse_with_server_name(
registration
.get("sender_localpart")
.unwrap()
.as_str()
.unwrap(),
db.globals.server_name(),
)
.unwrap()
},
|string| {
UserId::try_from(string.expect("parsing to string always works"))
.unwrap()
},
);
if !db.users.exists(&user_id).unwrap() {
return Failure((Status::Unauthorized, ()));
}
// TODO: Check if appservice is allowed to be that user
(Some(user_id), None, true)
}
AuthScheme::ServerSignatures => (None, None, true),
AuthScheme::None => (None, None, true),
}
} else {
match T::METADATA.authentication {
AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
if let Some(token) = token {
match db.users.find_from_token(&token).unwrap() {
// TODO: M_UNKNOWN_TOKEN
None => return Failure((Status::Unauthorized, ())),
Some((user_id, device_id)) => {
(Some(user_id), Some(device_id.into()), false)
}
}
} else {
// TODO: M_MISSING_TOKEN
return Failure((Status::Unauthorized, ()));
}
}
AuthScheme::ServerSignatures => (None, None, false),
AuthScheme::None => (None, None, false),
}
_ => (None, None),
};
let mut http_request = http::Request::builder()
@ -103,7 +148,7 @@ where
handle.read_to_end(&mut body).await.unwrap();
let http_request = http_request.body(body.clone()).unwrap();
log::debug!("{:?}", http_request);
debug!("{:?}", http_request);
match <T as Outgoing>::Incoming::try_from(http_request) {
Ok(t) => Success(Ruma {
@ -114,6 +159,7 @@ where
json_body: utils::string_from_bytes(&body)
.ok()
.and_then(|s| serde_json::value::RawValue::from_string(s).ok()),
from_appservice,
}),
Err(e) => {
warn!("{:?}", e);

189
src/server_server.rs
View file

@ -1,10 +1,7 @@
use crate::{
client_server, database::rooms::ClosestParent, utils, ConduitResult, Database, Error, PduEvent,
Result, Ruma,
};
use crate::{client_server, utils, ConduitResult, Database, Error, PduEvent, Result, Ruma};
use get_profile_information::v1::ProfileField;
use http::header::{HeaderValue, AUTHORIZATION, HOST};
use log::{error, warn};
use log::warn;
use rocket::{get, post, put, response::content::Json, State};
use ruma::{
api::{
@ -27,7 +24,6 @@ use std::{
collections::BTreeMap,
convert::TryFrom,
fmt::Debug,
sync::Arc,
time::{Duration, SystemTime},
};
@ -73,7 +69,6 @@ where
.cloned();
let (actual_destination, host) = if let Some(result) = maybe_result {
println!("Loaded {} -> {:?}", destination, result);
result
} else {
let result = find_actual_destination(globals, &destination).await;
@ -82,7 +77,6 @@ where
.write()
.unwrap()
.insert(destination.clone(), result.clone());
println!("Saving {} -> {:?}", destination, result);
result
};
@ -491,173 +485,28 @@ pub async fn send_transaction_message_route<'a>(
continue;
}
// If it is not a state event, we can skip state-res... maybe
if value.get("state_key").is_none() {
if !db.rooms.is_joined(&pdu.sender, room_id)? {
warn!("Sender is not joined {}", pdu.kind);
resolved_map.insert(event_id, Err("User is not in this room".into()));
continue;
}
let count = db.globals.next_count()?;
let mut pdu_id = room_id.as_bytes().to_vec();
pdu_id.push(0xff);
pdu_id.extend_from_slice(&count.to_be_bytes());
let count = db.globals.next_count()?;
let mut pdu_id = room_id.as_bytes().to_vec();
pdu_id.push(0xff);
pdu_id.extend_from_slice(&count.to_be_bytes());
db.rooms.append_to_state(&pdu_id, &pdu)?;
db.rooms.append_to_state(&pdu_id, &pdu)?;
db.rooms.append_pdu(
&pdu,
&value,
count,
pdu_id.clone().into(),
&db.globals,
&db.account_data,
&db.admin,
)?;
db.rooms.append_pdu(
&pdu,
&value,
count,
pdu_id.into(),
&db.globals,
&db.account_data,
&db.admin,
)?;
resolved_map.insert(event_id, Ok::<(), String>(()));
continue;
for appservice in db.appservice.iter_all().filter_map(|r| r.ok()) {
db.sending.send_pdu_appservice(&appservice.0, &pdu_id)?;
}
// We have a state event so we need info for state-res
let get_state_response = match send_request(
&db.globals,
body.body.origin.clone(),
ruma::api::federation::event::get_room_state::v1::Request {
room_id,
event_id: &event_id,
},
)
.await
{
Ok(res) => res,
// We can't hard fail because there are some valid errors, just
// keep checking PDU's
//
// As an example a possible error
// {"errcode":"M_FORBIDDEN","error":"Host not in room."}
Err(err) => {
resolved_map.insert(event_id, Err(err.to_string()));
continue;
}
};
let their_current_state = get_state_response
.pdus
.iter()
.chain(get_state_response.auth_chain.iter()) // add auth events
.map(|pdu| {
let (event_id, json) = crate::pdu::process_incoming_pdu(pdu);
(
event_id.clone(),
Arc::new(
// When creating a StateEvent the event_id arg will be used
// over any found in the json and it will not use ruma::reference_hash
// to generate one
state_res::StateEvent::from_id_canon_obj(event_id, json)
.expect("valid pdu json"),
),
)
})
.collect::<BTreeMap<_, _>>();
let our_current_state = db.rooms.room_state_full(room_id)?;
// State resolution takes care of these checks
// 4. Passes authorization rules based on the event's auth events, otherwise it is rejected.
// 5. Passes authorization rules based on the state at the event, otherwise it is rejected.
// TODO: 6. Passes authorization rules based on the current state of the room, otherwise it is "soft failed".
match state_res::StateResolution::resolve(
room_id,
&ruma::RoomVersionId::Version6,
&[
our_current_state
.iter()
.map(|((ev, sk), v)| ((ev.clone(), sk.to_owned()), v.event_id.clone()))
.collect::<BTreeMap<_, _>>(),
their_current_state
.iter()
.map(|(_id, v)| ((v.kind(), v.state_key()), v.event_id()))
.collect::<BTreeMap<_, _>>(),
],
Some(
our_current_state
.iter()
.map(|(_k, v)| (v.event_id.clone(), v.convert_for_state_res()))
.chain(
their_current_state
.iter()
.map(|(id, ev)| (id.clone(), ev.clone())),
)
.collect::<BTreeMap<_, _>>(),
),
&db.rooms,
) {
Ok(resolved) if resolved.values().any(|id| &event_id == id) => {
// If the event is older than the last event in pduid_pdu Tree then find the
// closest ancestor we know of and insert after the known ancestor by
// altering the known events pduid to = same roomID + same count bytes + 0x1
// pushing a single byte every time a simple append cannot be done.
match db.rooms.get_latest_pduid_before(
room_id,
&pdu.prev_events,
&their_current_state,
)? {
Some(ClosestParent::Append) => {
let count = db.globals.next_count()?;
let mut pdu_id = room_id.as_bytes().to_vec();
pdu_id.push(0xff);
pdu_id.extend_from_slice(&count.to_be_bytes());
db.rooms.append_pdu(
&pdu,
&value,
count,
pdu_id.into(),
&db.globals,
&db.account_data,
&db.admin,
)?;
}
Some(ClosestParent::Insert(old_count)) => {
let count = old_count;
let mut pdu_id = room_id.as_bytes().to_vec();
pdu_id.push(0xff);
pdu_id.extend_from_slice(&count.to_be_bytes());
// Create a new count that is after old_count but before
// the pdu appended after
pdu_id.push(1);
db.rooms.append_pdu(
&pdu,
&value,
count,
pdu_id.into(),
&db.globals,
&db.account_data,
&db.admin,
)?;
}
_ => {
error!("Not a sequential event or no parents found");
continue;
}
}
resolved_map.insert(event_id, Ok::<(), String>(()));
}
// If the eventId is not found in the resolved state auth has failed
Ok(_) => {
resolved_map.insert(
event_id,
Err("This event failed authentication, not found in resolved set".into()),
);
}
Err(e) => {
resolved_map.insert(event_id, Err(e.to_string()));
}
};
resolved_map.insert(event_id, Ok::<(), String>(()));
}
Ok(send_transaction_message::v1::Response { pdus: resolved_map }.into())