allow adding extra signing keys

We want to make sure users know this sliding sync impl is pretty buggy
before they attempt to use it.

Cargo.toml

@@ -5,7 +5,6 @@ explicit_outlives_requirements = "warn"
 macro_use_extern_crate = "warn"
 missing_abi = "warn"
 noop_method_call = "warn"
-pointer_structural_match = "warn"
 single_use_lifetimes = "warn"
 unreachable_pub = "warn"
 unsafe_op_in_unsafe_fn = "warn"
@@ -17,7 +16,7 @@ unused_qualifications = "warn"
 
 [workspace.lints.clippy]
 # Groups. Keep alphabetically sorted
-pedantic = "warn"
+pedantic = { level = "warn", priority = -1 }
 
 # Lints. Keep alphabetically sorted
 as_conversions = "warn"
@@ -80,7 +79,7 @@ version = "0.1.0"
 edition = "2021"
 
 # See also `rust-toolchain.toml`
-rust-version = "1.78.0"
+rust-version = "1.81.0"
 
 [lints]
 workspace = true

book/changelog.md

@@ -220,3 +220,5 @@ This will be the first release of Grapevine since it was forked from Conduit
     [!84](https://gitlab.computer.surgery/matrix/grapevine-fork/-/merge_requests/84))
 15. Added support for Authenticated Media ([MSC3916](https://github.com/matrix-org/matrix-spec-proposals/pull/3916)).
     ([!58](https://gitlab.computer.surgery/matrix/grapevine-fork/-/merge_requests/58))
+16. Added support for configuring and serving `/.well-known/matrix/...` data.
+    ([!90](https://gitlab.computer.surgery/matrix/grapevine-fork/-/merge_requests/90))

flake.lock

@@ -250,13 +250,13 @@
     "rust-manifest": {
       "flake": false,
       "locked": {
-        "narHash": "sha256-aZFye4UrtlcvLHrISldx4g9uGt3thDbVlLMK5keBSj0=",
+        "narHash": "sha256-tB9BZB6nRHDk5ELIVlGYlIjViLKBjQl52nC1avhcCwA=",
         "type": "file",
-        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
       },
       "original": {
         "type": "file",
-        "url": "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml"
+        "url": "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml"
       }
     },
     "systems": {

flake.nix

@@ -11,7 +11,7 @@
 
     rust-manifest = {
       # Keep version in sync with rust-toolchain.toml
-      url = "https://static.rust-lang.org/dist/channel-rust-1.78.0.toml";
+      url = "https://static.rust-lang.org/dist/channel-rust-1.81.0.toml";
       flake = false;
     };
   };
@@ -70,8 +70,24 @@
         };
       in
       {
-        packages = {
+        packages = rec {
           default = (mkScope pkgs).default;
+          oci-image = pkgs.dockerTools.buildImage {
+            name = default.pname;
+            tag = "next";
+            copyToRoot = [
+              pkgs.dockerTools.caCertificates
+            ];
+            config = {
+              # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT)
+              # are handled as expected
+              Entrypoint = [
+                "${pkgs.lib.getExe' pkgs.tini "tini"}"
+                "--"
+                "${pkgs.lib.getExe default}"
+              ];
+            };
+          };
         }
         //
         builtins.listToAttrs

rust-toolchain.toml

@@ -9,7 +9,7 @@
 # If you're having trouble making the relevant changes, bug a maintainer.
 
 [toolchain]
-channel = "1.78.0"
+channel = "1.81.0"
 components = [
     # For rust-analyzer
     "rust-src",

src/api.rs

@@ -2,3 +2,4 @@ pub(crate) mod appservice_server;
 pub(crate) mod client_server;
 pub(crate) mod ruma_wrapper;
 pub(crate) mod server_server;
+pub(crate) mod well_known;

src/api/client_server/account.rs

@@ -316,8 +316,7 @@ pub(crate) async fn register_route(
 /// - Requires UIAA to verify user password
 /// - Changes the password of the sender user
 /// - The password hash is calculated using argon2 with 32 character salt, the
-///   plain password is
-/// not saved
+///   plain password is not saved
 ///
 /// If `logout_devices` is true it does the following for each device except the
 /// sender device:

src/api/client_server/context.rs

@@ -16,8 +16,7 @@ use crate::{services, Ar, Error, Ra, Result};
 /// Allows loading room history around an event.
 ///
 /// - Only works if the user is joined (TODO: always allow, but only show events
-///   if the user was
-/// joined, depending on `history_visibility`)
+///   if the user was joined, depending on `history_visibility`)
 #[allow(clippy::too_many_lines)]
 pub(crate) async fn get_context_route(
     body: Ar<get_context::v3::Request>,

src/api/client_server/message.rs

@@ -117,8 +117,7 @@ pub(crate) async fn send_message_event_route(
 /// Allows paginating through room history.
 ///
 /// - Only works if the user is joined (TODO: always allow, but only show events
-///   where the user was
-/// joined, depending on `history_visibility`)
+///   where the user was joined, depending on `history_visibility`)
 #[allow(clippy::too_many_lines)]
 pub(crate) async fn get_message_events_route(
     body: Ar<get_message_events::v3::Request>,

src/api/client_server/sync.rs

@@ -36,8 +36,7 @@ use crate::{
 /// Synchronize the client's state with the latest state on the server.
 ///
 /// - This endpoint takes a `since` parameter which should be the `next_batch`
-///   value from a
-/// previous request for incremental syncs.
+///   value from a previous request for incremental syncs.
 ///
 /// Calling this endpoint without a `since` parameter returns:
 /// - Some of the most recent events of each timeline
@@ -50,11 +49,9 @@ use crate::{
 /// - Some of the most recent events of each timeline that happened after
 ///   `since`
 /// - If user joined the room after `since`: All state events (unless lazy
-///   loading is activated) and
-/// all device list updates in that room
+///   loading is activated) and all device list updates in that room
 /// - If the user was already in the room: A list of all events that are in the
-///   state now, but were
-/// not in the state at `since`
+///   state now, but were not in the state at `since`
 /// - If the state we send contains a member event: Joined and invited member
 ///   counts, heroes
 /// - Device list updates that happened after `since`

src/api/client_server/user_directory.rs

@@ -13,8 +13,7 @@ use crate::{services, Ar, Ra, Result};
 /// Searches all known users for a match.
 ///
 /// - Hides any local users that aren't in any public rooms (i.e. those that
-///   have the join rule set to public)
-/// and don't share a room with the sender
+///   have the join rule set to public) and don't share a room with the sender
 pub(crate) async fn search_users_route(
     body: Ar<search_users::v3::Request>,
 ) -> Result<Ra<search_users::v3::Response>> {

src/api/server_server.rs

@@ -64,7 +64,7 @@ use ruma::{
 };
 use serde_json::value::{to_raw_value, RawValue as RawJsonValue};
 use tokio::sync::RwLock;
-use tracing::{debug, error, field, warn};
+use tracing::{debug, error, field, trace, trace_span, warn};
 
 use super::appservice_server;
 use crate::{
@@ -253,6 +253,14 @@ where
         signature,
     )));
 
+    // can be enabled selectively using `filter =
+    // grapevine[outgoing_request_curl]=trace` in config
+    trace_span!("outgoing_request_curl").in_scope(|| {
+        trace!(
+            cmd = utils::curlify(&http_request),
+            "curl command line for outgoing request"
+        );
+    });
     let reqwest_request = reqwest::Request::try_from(http_request)?;
 
     let url = reqwest_request.url().clone();
@@ -532,7 +540,7 @@ async fn request_well_known(destination: &str) -> Option<String> {
     let response = services()
         .globals
         .default_client()
-        .get(&format!("https://{destination}/.well-known/matrix/server"))
+        .get(format!("https://{destination}/.well-known/matrix/server"))
         .send()
         .await;
     debug!("Got well known response");
@@ -565,23 +573,26 @@ pub(crate) async fn get_server_version_route(
 /// Gets the public signing keys of this server.
 ///
 /// - Matrix does not support invalidating public keys, so the key returned by
-///   this will be valid
-/// forever.
+///   this will be valid forever.
 // Response type for this endpoint is Json because we need to calculate a
 // signature for the response
 pub(crate) async fn get_server_keys_route() -> Result<impl IntoResponse> {
+    let keys: Vec<_> = [services().globals.keypair()]
+        .into_iter()
+        .chain(&services().globals.config.extra_key)
+        .collect();
     let mut verify_keys: BTreeMap<OwnedServerSigningKeyId, VerifyKey> =
         BTreeMap::new();
-    verify_keys.insert(
-        format!("ed25519:{}", services().globals.keypair().version())
-            .try_into()
-            .expect("found invalid server signing keys in DB"),
-        VerifyKey {
-            key: Base64::new(
-                services().globals.keypair().public_key().to_vec(),
-            ),
-        },
-    );
+    for key in &keys {
+        verify_keys.insert(
+            format!("ed25519:{}", key.version())
+                .try_into()
+                .expect("found invalid server signing keys in DB"),
+            VerifyKey {
+                key: Base64::new(key.public_key().to_vec()),
+            },
+        );
+    }
     let mut response = serde_json::from_slice(
         get_server_keys::v2::Response {
             server_key: Raw::new(&ServerSigningKeys {
@@ -602,12 +613,14 @@ pub(crate) async fn get_server_keys_route() -> Result<impl IntoResponse> {
     )
     .unwrap();
 
-    ruma::signatures::sign_json(
-        services().globals.server_name().as_str(),
-        services().globals.keypair(),
-        &mut response,
-    )
-    .unwrap();
+    for key in &keys {
+        ruma::signatures::sign_json(
+            services().globals.server_name().as_str(),
+            *key,
+            &mut response,
+        )
+        .unwrap();
+    }
 
     Ok(Json(response))
 }
@@ -617,8 +630,7 @@ pub(crate) async fn get_server_keys_route() -> Result<impl IntoResponse> {
 /// Gets the public signing keys of this server.
 ///
 /// - Matrix does not support invalidating public keys, so the key returned by
-///   this will be valid
-/// forever.
+///   this will be valid forever.
 pub(crate) async fn get_server_keys_deprecated_route() -> impl IntoResponse {
     get_server_keys_route().await
 }

src/api/well_known.rs

@@ -0,0 +1,64 @@
+#![warn(missing_docs, clippy::missing_docs_in_private_items)]
+
+//! Handle requests for `/.well-known/matrix/...` files
+
+use http::StatusCode;
+use ruma::api::{
+    client::discovery::discover_homeserver as client,
+    federation::discovery::discover_homeserver as server,
+};
+
+use crate::{services, Ar, Ra};
+
+/// Handler for `/.well-known/matrix/server`
+pub(crate) async fn server(
+    _: Ar<server::Request>,
+) -> Result<Ra<server::Response>, StatusCode> {
+    let Some(authority) =
+        services().globals.config.server_discovery.server.authority.clone()
+    else {
+        return Err(StatusCode::NOT_FOUND);
+    };
+
+    if authority == services().globals.config.server_name {
+        // Delegation isn't needed in this case
+        return Err(StatusCode::NOT_FOUND);
+    }
+
+    Ok(Ra(server::Response::new(authority)))
+}
+
+/// Handler for `/.well-known/matrix/client`
+pub(crate) async fn client(_: Ar<client::Request>) -> Ra<client::Response> {
+    let authority = services()
+        .globals
+        .config
+        .server_discovery
+        .client
+        .authority
+        .clone()
+        .unwrap_or_else(|| services().globals.config.server_name.clone());
+
+    let scheme = if services().globals.config.server_discovery.client.insecure {
+        "http"
+    } else {
+        "https"
+    };
+
+    let base_url = format!("{scheme}://{authority}");
+
+    // I wish ruma used an actual URL type instead of `String`
+    Ra(client::Response {
+        homeserver: client::HomeserverInfo::new(base_url.clone()),
+        identity_server: None,
+        sliding_sync_proxy: services()
+            .globals
+            .config
+            .server_discovery
+            .client
+            .advertise_sliding_sync
+            .then_some(client::SlidingSyncProxyInfo {
+                url: base_url,
+            }),
+    })
+}

src/config.rs

@@ -6,8 +6,8 @@ use std::{
 };
 
 use once_cell::sync::Lazy;
-use ruma::{OwnedServerName, RoomVersionId};
-use serde::Deserialize;
+use ruma::{serde::Base64, signatures::Ed25519KeyPair, OwnedServerName, RoomVersionId};
+use serde::{Deserialize, Deserializer};
 
 use crate::error;
 
@@ -29,8 +29,16 @@ pub(crate) struct Config {
     #[serde(default = "default_listen")]
     pub(crate) listen: Vec<ListenConfig>,
     pub(crate) tls: Option<TlsConfig>,
+    #[serde(default, deserialize_with = "deserialize_keys_config")]
+    pub(crate) extra_key: Vec<Ed25519KeyPair>,
 
+    /// The name of this homeserver
+    ///
+    /// This is the value that will appear e.g. in user IDs and room aliases.
     pub(crate) server_name: OwnedServerName,
+
+    #[serde(default)]
+    pub(crate) server_discovery: ServerDiscovery,
     pub(crate) database: DatabaseConfig,
     #[serde(default)]
     pub(crate) federation: FederationConfig,
@@ -63,6 +71,54 @@ pub(crate) struct Config {
     pub(crate) emergency_password: Option<String>,
 }
 
+fn deserialize_keys_config<'de, D>(de: D) -> Result<Vec<Ed25519KeyPair>, D::Error> where D: Deserializer<'de> {
+    use serde::de::Error;
+    
+    #[derive(Debug, Deserialize)]
+    struct RawConfig {
+        key: Base64,
+        version: String,
+    }
+
+    let raw: Vec<RawConfig> = Deserialize::deserialize(de)?;
+    raw
+        .into_iter()
+        .map(|r| Ed25519KeyPair::from_der(&r.key.into_inner(), r.version).map_err(D::Error::custom))
+        .collect()
+}
+
+#[derive(Debug, Default, Deserialize)]
+pub(crate) struct ServerDiscovery {
+    /// Server-server discovery configuration
+    #[serde(default)]
+    pub(crate) server: ServerServerDiscovery,
+
+    /// Client-server discovery configuration
+    #[serde(default)]
+    pub(crate) client: ClientServerDiscovery,
+}
+
+/// Server-server discovery configuration
+#[derive(Debug, Default, Deserialize)]
+pub(crate) struct ServerServerDiscovery {
+    /// The alternative authority to make server-server API requests to
+    pub(crate) authority: Option<OwnedServerName>,
+}
+
+/// Client-server discovery configuration
+#[derive(Debug, Default, Deserialize)]
+pub(crate) struct ClientServerDiscovery {
+    /// The alternative authority to make client-server API requests to
+    pub(crate) authority: Option<OwnedServerName>,
+
+    /// Controls whether HTTPS is used
+    #[serde(default)]
+    pub(crate) insecure: bool,
+
+    #[serde(default, rename = "advertise_buggy_sliding_sync")]
+    pub(crate) advertise_sliding_sync: bool,
+}
+
 #[derive(Debug, Deserialize)]
 pub(crate) struct TlsConfig {
     pub(crate) certs: String,

src/database/key_value/rooms/edus/read_receipt.rs

@@ -1,5 +1,3 @@
-use std::mem;
-
 use ruma::{
     events::receipt::ReceiptEvent, serde::Raw, CanonicalJsonObject,
     OwnedUserId, RoomId, UserId,
@@ -83,7 +81,7 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
                 .take_while(move |(k, _)| k.starts_with(&prefix2))
                 .map(move |(k, v)| {
                     let count = utils::u64_from_bytes(
-                        &k[prefix.len()..prefix.len() + mem::size_of::<u64>()],
+                        &k[prefix.len()..prefix.len() + size_of::<u64>()],
                     )
                     .map_err(|_| {
                         Error::bad_database(
@@ -92,7 +90,7 @@ impl service::rooms::edus::read_receipt::Data for KeyValueDatabase {
                     })?;
                     let user_id = UserId::parse(
                         utils::string_from_bytes(
-                            &k[prefix.len() + mem::size_of::<u64>() + 1..],
+                            &k[prefix.len() + size_of::<u64>() + 1..],
                         )
                         .map_err(|_| {
                             Error::bad_database(

src/database/key_value/rooms/pdu_metadata.rs

@@ -1,4 +1,4 @@
-use std::{mem, sync::Arc};
+use std::sync::Arc;
 
 use ruma::{EventId, RoomId, UserId};
 
@@ -47,12 +47,13 @@ impl service::rooms::pdu_metadata::Data for KeyValueDatabase {
                 .iter_from(&current, true)
                 .take_while(move |(k, _)| k.starts_with(&prefix))
                 .map(move |(tofrom, _data)| {
-                    let from = utils::u64_from_bytes(
-                        &tofrom[(mem::size_of::<u64>())..],
-                    )
-                    .map_err(|_| {
-                        Error::bad_database("Invalid count in tofrom_relation.")
-                    })?;
+                    let from =
+                        utils::u64_from_bytes(&tofrom[(size_of::<u64>())..])
+                            .map_err(|_| {
+                                Error::bad_database(
+                                    "Invalid count in tofrom_relation.",
+                                )
+                            })?;
 
                     let mut pduid = shortroomid.get().to_be_bytes().to_vec();
                     pduid.extend_from_slice(&from.to_be_bytes());

src/database/key_value/rooms/threads.rs

@@ -1,5 +1,3 @@
-use std::mem;
-
 use ruma::{
     api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId,
     UserId,
@@ -36,14 +34,13 @@ impl service::rooms::threads::Data for KeyValueDatabase {
                 .iter_from(&current, true)
                 .take_while(move |(k, _)| k.starts_with(&prefix))
                 .map(move |(pduid, _users)| {
-                    let count = utils::u64_from_bytes(
-                        &pduid[(mem::size_of::<u64>())..],
-                    )
-                    .map_err(|_| {
-                        Error::bad_database(
-                            "Invalid pduid in threadid_userids.",
-                        )
-                    })?;
+                    let count =
+                        utils::u64_from_bytes(&pduid[(size_of::<u64>())..])
+                            .map_err(|_| {
+                                Error::bad_database(
+                                    "Invalid pduid in threadid_userids.",
+                                )
+                            })?;
 
                     let pduid = PduId::new(pduid);
 

src/main.rs

@@ -50,7 +50,7 @@ mod service;
 mod utils;
 
 pub(crate) use api::ruma_wrapper::{Ar, Ra};
-use api::{client_server, server_server};
+use api::{client_server, server_server, well_known};
 pub(crate) use config::{Config, ListenConfig};
 pub(crate) use database::KeyValueDatabase;
 pub(crate) use service::{pdu::PduEvent, Services};
@@ -147,26 +147,42 @@ async fn run_server() -> Result<(), error::Serve> {
     let middlewares = ServiceBuilder::new()
         .sensitive_headers([header::AUTHORIZATION])
         .layer(axum::middleware::from_fn(spawn_task))
-        .layer(TraceLayer::new_for_http().make_span_with(
-            |request: &http::Request<_>| {
-                let endpoint = if let Some(endpoint) =
-                    request.extensions().get::<MatchedPath>()
-                {
-                    endpoint.as_str()
-                } else {
-                    request.uri().path()
-                };
+        .layer(
+            TraceLayer::new_for_http()
+                .make_span_with(|request: &http::Request<_>| {
+                    let endpoint = if let Some(endpoint) =
+                        request.extensions().get::<MatchedPath>()
+                    {
+                        endpoint.as_str()
+                    } else {
+                        request.uri().path()
+                    };
 
-                let method = request.method();
+                    let method = request.method();
 
-                tracing::info_span!(
-                    "http_request",
-                    otel.name = format!("{method} {endpoint}"),
-                    %method,
-                    %endpoint,
-                )
-            },
-        ))
+                    tracing::info_span!(
+                        "http_request",
+                        otel.name = format!("{method} {endpoint}"),
+                        %method,
+                        %endpoint,
+                    )
+                })
+                .on_request(
+                    |request: &http::Request<_>, _span: &tracing::Span| {
+                        // can be enabled selectively using `filter =
+                        // grapevine[incoming_request_curl]=trace` in config
+                        tracing::trace_span!("incoming_request_curl").in_scope(
+                            || {
+                                tracing::trace!(
+                                    cmd = utils::curlify(request),
+                                    "curl command line for incoming request \
+                                     (guessed hostname)"
+                                );
+                            },
+                        );
+                    },
+                ),
+        )
         .layer(axum::middleware::from_fn(unrecognized_method))
         .layer(
             CorsLayer::new()
@@ -475,6 +491,10 @@ fn routes(config: &Config) -> Router {
         .route("/", get(it_works))
         .fallback(not_found);
 
+    let router = router
+        .route("/.well-known/matrix/client", get(well_known::client))
+        .route("/.well-known/matrix/server", get(well_known::server));
+
     if config.federation.enable {
         router
             .ruma_route(s2s::get_server_version_route)

src/service/media.rs

@@ -171,23 +171,23 @@ impl Service {
                         / u64::from(original_height)
                 };
                 if use_width {
-                    if intermediate <= u64::from(::std::u32::MAX) {
-                        (width, intermediate.try_into().unwrap_or(u32::MAX))
+                    if let Ok(intermediate) = u32::try_from(intermediate) {
+                        (width, intermediate)
                     } else {
                         (
-                            (u64::from(width) * u64::from(::std::u32::MAX)
+                            (u64::from(width) * u64::from(u32::MAX)
                                 / intermediate)
                                 .try_into()
                                 .unwrap_or(u32::MAX),
-                            ::std::u32::MAX,
+                            u32::MAX,
                         )
                     }
-                } else if intermediate <= u64::from(::std::u32::MAX) {
-                    (intermediate.try_into().unwrap_or(u32::MAX), height)
+                } else if let Ok(intermediate) = u32::try_from(intermediate) {
+                    (intermediate, height)
                 } else {
                     (
-                        ::std::u32::MAX,
-                        (u64::from(height) * u64::from(::std::u32::MAX)
+                        u32::MAX,
+                        (u64::from(height) * u64::from(u32::MAX)
                             / intermediate)
                             .try_into()
                             .unwrap_or(u32::MAX),

src/utils.rs

@@ -306,6 +306,66 @@ impl<'a> TryFrom<&'a MxcUri> for MxcData<'a> {
     }
 }
 
+fn curlify_args<T>(req: &http::Request<T>) -> Option<Vec<String>> {
+    let mut args =
+        vec!["curl".to_owned(), "-X".to_owned(), req.method().to_string()];
+
+    for (name, val) in req.headers() {
+        args.extend([
+            "-H".to_owned(),
+            format!("{name}: {}", val.to_str().ok()?),
+        ]);
+    }
+
+    let fix_uri = || {
+        if req.uri().scheme().is_some() {
+            return None;
+        }
+        if req.uri().authority().is_some() {
+            return None;
+        }
+        let mut parts = req.uri().clone().into_parts();
+
+        parts.scheme = Some(http::uri::Scheme::HTTPS);
+
+        let host =
+            req.headers().get(http::header::HOST)?.to_str().ok()?.to_owned();
+        parts.authority =
+            Some(http::uri::Authority::from_maybe_shared(host).ok()?);
+
+        http::uri::Uri::from_parts(parts).ok()
+    };
+
+    let uri = if let Some(new_uri) = fix_uri() {
+        Cow::Owned(new_uri)
+    } else {
+        Cow::Borrowed(req.uri())
+    };
+
+    args.push(uri.to_string());
+
+    Some(args)
+}
+
+pub(crate) fn curlify<T>(req: &http::Request<T>) -> Option<String> {
+    let args = curlify_args(req)?;
+
+    Some(
+        args.into_iter()
+            .map(|arg| {
+                if arg.chars().all(|c| {
+                    c.is_alphanumeric() || ['-', '_', ':', '/'].contains(&c)
+                }) {
+                    arg
+                } else {
+                    format!("'{}'", arg.replace('\'', "\\'"))
+                }
+            })
+            .collect::<Vec<_>>()
+            .join(" "),
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use crate::utils::dbg_truncate_str;