nero/tsuki
1
0
Fork 0
Code Issues 2 Pull requests Projects Releases Packages Wiki Activity Actions

feat: ask for backfill

Browse source
This commit is contained in:
Timo Kösters 2023-02-20 22:59:45 +01:00
parent 23b18d71ee
commit 7bdd9660aa
No known key found for this signature in database
GPG key ID: 0B25E636FBA7E4CB
12 changed files with 502 additions and 269 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/api/client_server/context.rs
View file

@ -27,25 +27,24 @@ pub async fn get_context_route(
let mut lazy_loaded = HashSet::new();
let base_pdu_id = services()
let base_token = services()
.rooms
.timeline
.get_pdu_id(&body.event_id)?
.get_pdu_count(&body.event_id)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"Base event id not found.",
))?;
let base_token = services().rooms.timeline.pdu_count(&base_pdu_id)?;
let base_event = services()
.rooms
.timeline
.get_pdu_from_id(&base_pdu_id)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"Base event not found.",
))?;
let base_event =
services()
.rooms
.timeline
.get_pdu(&body.event_id)?
.ok_or(Error::BadRequest(
ErrorKind::NotFound,
"Base event not found.",
))?;
let room_id = base_event.room_id.clone();
@ -97,10 +96,7 @@ pub async fn get_context_route(
}
}
let start_token = events_before
.last()
.and_then(|(pdu_id, _)| services().rooms.timeline.pdu_count(pdu_id).ok())
.map(|count| count.to_string());
let start_token = events_before.last().map(|(count, _)| count.stringify());
let events_before: Vec<_> = events_before
.into_iter()
@ -151,10 +147,7 @@ pub async fn get_context_route(
.state_full_ids(shortstatehash)
.await?;
let end_token = events_after
.last()
.and_then(|(pdu_id, _)| services().rooms.timeline.pdu_count(pdu_id).ok())
.map(|count| count.to_string());
let end_token = events_after.last().map(|(count, _)| count.stringify());
let events_after: Vec<_> = events_after
.into_iter()

6
src/api/client_server/membership.rs
View file

@ -714,8 +714,10 @@ async fn join_room_by_id_helper(
.ok()?
},
)
.map_err(|_e| Error::BadRequest(ErrorKind::InvalidParam, "Auth check failed"))?
{
.map_err(|e| {
warn!("Auth check failed: {e}");
Error::BadRequest(ErrorKind::InvalidParam, "Auth check failed")
})? {
return Err(Error::BadRequest(
ErrorKind::InvalidParam,
"Auth check failed",

52
src/api/client_server/message.rs
View file

@ -1,4 +1,7 @@
use crate::{service::pdu::PduBuilder, services, utils, Error, Result, Ruma};
use crate::{
service::{pdu::PduBuilder, rooms::timeline::PduCount},
services, utils, Error, Result, Ruma,
};
use ruma::{
api::client::{
error::ErrorKind,
@ -122,17 +125,17 @@ pub async fn get_message_events_route(
}
let from = match body.from.clone() {
Some(from) => from
.parse()
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from` value."))?,
Some(from) => PduCount::try_from_string(&from)?,
None => match body.dir {
ruma::api::client::Direction::Forward => 0,
ruma::api::client::Direction::Backward => u64::MAX,
ruma::api::client::Direction::Forward => PduCount::min(),
ruma::api::client::Direction::Backward => PduCount::max(),
},
};
let to = body.to.as_ref().map(|t| t.parse());
let to = body
.to
.as_ref()
.and_then(|t| PduCount::try_from_string(&t).ok());
services().rooms.lazy_loading.lazy_load_confirm_delivery(
sender_user,
@ -158,15 +161,7 @@ pub async fn get_message_events_route(
.pdus_after(sender_user, &body.room_id, from)?
.take(limit)
.filter_map(|r| r.ok()) // Filter out buggy events
.filter_map(|(pdu_id, pdu)| {
services()
.rooms
.timeline
.pdu_count(&pdu_id)
.map(|pdu_count| (pdu_count, pdu))
.ok()
})
.take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to`
.take_while(|&(k, _)| Some(k) != to) // Stop at `to`
.collect();
for (_, event) in &events_after {
@ -192,26 +187,23 @@ pub async fn get_message_events_route(
.map(|(_, pdu)| pdu.to_room_event())
.collect();
resp.start = from.to_string();
resp.end = next_token.map(|count| count.to_string());
resp.start = from.stringify();
resp.end = next_token.map(|count| count.stringify());
resp.chunk = events_after;
}
ruma::api::client::Direction::Backward => {
services()
.rooms
.timeline
.backfill_if_required(&body.room_id, from)
.await?;
let events_before: Vec<_> = services()
.rooms
.timeline
.pdus_until(sender_user, &body.room_id, from)?
.take(limit)
.filter_map(|r| r.ok()) // Filter out buggy events
.filter_map(|(pdu_id, pdu)| {
services()
.rooms
.timeline
.pdu_count(&pdu_id)
.map(|pdu_count| (pdu_count, pdu))
.ok()
})
.take_while(|&(k, _)| Some(Ok(k)) != to) // Stop at `to`
.take_while(|&(k, _)| Some(k) != to) // Stop at `to`
.collect();
for (_, event) in &events_before {
@ -237,8 +229,8 @@ pub async fn get_message_events_route(
.map(|(_, pdu)| pdu.to_room_event())
.collect();
resp.start = from.to_string();
resp.end = next_token.map(|count| count.to_string());
resp.start = from.stringify();
resp.end = next_token.map(|count| count.stringify());
resp.chunk = events_before;
}
}

60
src/api/client_server/read_marker.rs
View file

@ -1,4 +1,4 @@
use crate::{services, Error, Result, Ruma};
use crate::{service::rooms::timeline::PduCount, services, Error, Result, Ruma};
use ruma::{
api::client::{error::ErrorKind, read_marker::set_read_marker, receipt::create_receipt},
events::{
@ -42,18 +42,28 @@ pub async fn set_read_marker_route(
}
if let Some(event) = &body.private_read_receipt {
services().rooms.edus.read_receipt.private_read_set(
&body.room_id,
sender_user,
services()
.rooms
.timeline
.get_pdu_count(event)?
.ok_or(Error::BadRequest(
let count = services()
.rooms
.timeline
.get_pdu_count(event)?
.ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Event does not exist.",
))?;
let count = match count {
PduCount::Backfilled(_) => {
return Err(Error::BadRequest(
ErrorKind::InvalidParam,
"Event does not exist.",
))?,
)?;
"Read receipt is in backfilled timeline",
))
}
PduCount::Normal(c) => c,
};
services()
.rooms
.edus
.read_receipt
.private_read_set(&body.room_id, sender_user, count)?;
}
if let Some(event) = &body.read_receipt {
@ -142,17 +152,27 @@ pub async fn create_receipt_route(
)?;
}
create_receipt::v3::ReceiptType::ReadPrivate => {
let count = services()
.rooms
.timeline
.get_pdu_count(&body.event_id)?
.ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Event does not exist.",
))?;
let count = match count {
PduCount::Backfilled(_) => {
return Err(Error::BadRequest(
ErrorKind::InvalidParam,
"Read receipt is in backfilled timeline",
))
}
PduCount::Normal(c) => c,
};
services().rooms.edus.read_receipt.private_read_set(
&body.room_id,
sender_user,
services()
.rooms
.timeline
.get_pdu_count(&body.event_id)?
.ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Event does not exist.",
))?,
count,
)?;
}
_ => return Err(Error::bad_database("Unsupported receipt type")),

34
src/api/client_server/sync.rs
View file

@ -1,4 +1,4 @@
use crate::{services, Error, Result, Ruma, RumaResponse};
use crate::{service::rooms::timeline::PduCount, services, Error, Result, Ruma, RumaResponse};
use ruma::{
api::client::{
filter::{FilterDefinition, LazyLoadOptions},
@ -172,6 +172,7 @@ async fn sync_helper(
let watcher = services().globals.watch(&sender_user, &sender_device);
let next_batch = services().globals.current_count()?;
let next_batchcount = PduCount::Normal(next_batch);
let next_batch_string = next_batch.to_string();
// Load filter
@ -197,6 +198,7 @@ async fn sync_helper(
.clone()
.and_then(|string| string.parse().ok())
.unwrap_or(0);
let sincecount = PduCount::Normal(since);
let mut presence_updates = HashMap::new();
let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in
@ -241,12 +243,12 @@ async fn sync_helper(
.rooms
.timeline
.last_timeline_count(&sender_user, &room_id)?
> since
> sincecount
{
let mut non_timeline_pdus = services()
.rooms
.timeline
.pdus_until(&sender_user, &room_id, u64::MAX)?
.pdus_until(&sender_user, &room_id, PduCount::max())?
.filter_map(|r| {
// Filter out buggy events
if r.is_err() {
@ -254,13 +256,7 @@ async fn sync_helper(
}
r.ok()
})
.take_while(|(pduid, _)| {
services()
.rooms
.timeline
.pdu_count(pduid)
.map_or(false, |count| count > since)
});
.take_while(|(pducount, _)| pducount > &sincecount);
// Take the last 10 events for the timeline
timeline_pdus = non_timeline_pdus
@ -295,7 +291,7 @@ async fn sync_helper(
&sender_user,
&sender_device,
&room_id,
since,
sincecount,
)?;
// Database queries:
@ -492,7 +488,7 @@ async fn sync_helper(
&sender_device,
&room_id,
lazy_loaded,
next_batch,
next_batchcount,
);
(
@ -582,7 +578,7 @@ async fn sync_helper(
&sender_device,
&room_id,
lazy_loaded,
next_batch,
next_batchcount,
);
let encrypted_room = services()
@ -711,10 +707,14 @@ async fn sync_helper(
let prev_batch = timeline_pdus
.first()
.map_or(Ok::<_, Error>(None), |(pdu_id, _)| {
Ok(Some(
services().rooms.timeline.pdu_count(pdu_id)?.to_string(),
))
.map_or(Ok::<_, Error>(None), |(pdu_count, _)| {
Ok(Some(match pdu_count {
PduCount::Backfilled(_) => {
error!("timeline in backfill state?!");
"0".to_owned()
}
PduCount::Normal(c) => c.to_string(),
}))
})?;
let room_events: Vec<_> = timeline_pdus

64
src/api/server_server.rs
View file

@ -629,6 +629,37 @@ pub async fn get_public_rooms_route(
})
}
pub fn parse_incoming_pdu(
pdu: &RawJsonValue,
) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> {
let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| {
warn!("Error parsing incoming event {:?}: {:?}", pdu, e);
Error::BadServerResponse("Invalid PDU in server response")
})?;
let room_id: OwnedRoomId = value
.get("room_id")
.and_then(|id| RoomId::parse(id.as_str()?).ok())
.ok_or(Error::BadRequest(
ErrorKind::InvalidParam,
"Invalid room id in pdu",
))?;
let room_version_id = services().rooms.state.get_room_version(&room_id)?;
let (event_id, value) = match gen_event_id_canonical_json(&pdu, &room_version_id) {
Ok(t) => t,
Err(_) => {
// Event could not be converted to canonical json
return Err(Error::BadRequest(
ErrorKind::InvalidParam,
"Could not convert event to canonical json.",
));
}
};
Ok((event_id, value, room_id))
}
/// # `PUT /_matrix/federation/v1/send/{txnId}`
///
/// Push EDUs and PDUs to this server.
@ -657,36 +688,7 @@ pub async fn send_transaction_message_route(
// let mut auth_cache = EventMap::new();
for pdu in &body.pdus {
let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| {
warn!("Error parsing incoming event {:?}: {:?}", pdu, e);
Error::BadServerResponse("Invalid PDU in server response")
})?;
let room_id: OwnedRoomId = match value
.get("room_id")
.and_then(|id| RoomId::parse(id.as_str()?).ok())
{
Some(id) => id,
None => {
// Event is invalid
continue;
}
};
let room_version_id = match services().rooms.state.get_room_version(&room_id) {
Ok(v) => v,
Err(_) => {
continue;
}
};
let (event_id, value) = match gen_event_id_canonical_json(pdu, &room_version_id) {
Ok(t) => t,
Err(_) => {
// Event could not be converted to canonical json
continue;
}
};
let (event_id, value, room_id) = parse_incoming_pdu(&pdu)?;
// We do not add the event_id field to the pdu here because of signature and hashes checks
services()
@ -1017,7 +1019,7 @@ pub async fn get_backfill_route(
Ok(true),
)
})
.map(|(pdu_id, _)| services().rooms.timeline.get_pdu_json_from_id(&pdu_id))
.map(|(_, pdu)| services().rooms.timeline.get_pdu_json(&pdu.event_id))
.filter_map(|r| r.ok().flatten())
.map(|pdu| PduEvent::convert_to_outgoing_federation_event(pdu))
.collect();