diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 7390c82..1e7ffbc 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -1,3 +1,7 @@ # Summary -- [Chapter 1](./chapter_1.md) +- [data](./data.md) +- [design](./design.md) +- [install](./install.md) +- [syncing](./syncing.md) +- [api](./api.md) diff --git a/src/api.md b/src/api.md new file mode 100644 index 0000000..80dad7e --- /dev/null +++ b/src/api.md @@ -0,0 +1,395 @@ +# api + +list of all api paths for client-server api + +each path is versioned on its own + +### endpoint summary + +WARNING: although a good portion of the api is taken by or inspired by +matrix, these endpoints aren't the same! Please read this documentation +rather than [spec.matrix.org](https://spec.matrix.org). + +NOTE: The api doesn't necessarily follow REST conventions. + +NOTE: This is the PLANNED api, and isn't what is currently +implemented. The endpoints and structs may change. + +#### client + +prefix = `/_jackwagon/client` + +The main apis for clients to interact with servers. Authentication will be outsourced to oidc. + +but QUERY isn't widely supported :( + +| method | path | description | +|--------|---------------------------------------------------|----------------------------------------------------| +| QUERY | /v1/info | Get info about this server | +| QUERY | /v1/sync | Receive new events; see [syncing](/docs/syncing) | +| QUERY | /v1/threads | Query for threads, get inbox | +| PUT | /v1/threads/participation | Set thread participation | +| POST | /v1/ack | Mark messages as read | +| GET | /v1/users/me | Get info about this user (profile) (whoami) | +| GET | /v1/users/{userId} | Get info about a user | +| PATCH | /v1/users/me | Set info about this user | +| PATCH | /v1/users/{userId} | Set info about a user | +| PUT | /v1/account-data/{key} | Write account data | +| GET | /v1/account-data/{key} | Read account data | +| POST | /v1/rooms/create | Create a room | +| PUT | /v1/rooms/membership | Change room membership for yourself and others | +| POST | /v1/rooms/{roomId}/send/{eventType} | Send an event | +| POST | /v1/rooms/{roomId}/ephemeral/{eventType} | Send an ephemeral event | +| PUT | /v1/rooms/{roomId}/state/{eventType}/{stateKey} | Send a state event, mutating the state of a room | +| GET | /v1/rooms/{roomId}/state/{eventType?}/{stateKey?} | Read state from a room | +| QUERY | /v1/rooms/{roomId}/events | Fetch events | +| QUERY | /v1/rooms/{roomId}/relations | Fetch relations | +| QUERY | /v1/rooms/{roomId}/aliases | Get aliases | +| PUT | /v1/rooms/{roomId}/aliases | Set aliases | +| PUT | /v1/rooms/{roomId}/account-data/{key} | Write account data | +| GET | /v1/rooms/{roomId}/account-data/{key} | Read account data | +| POST | /v1/aliases/resolve | Resolve aliases to room ids | +| QUERY | /v1/spaces/{roomId}/hierarchy | Fetch space children | +| QUERY | /v1/search/users | Search for users | +| QUERY | /v1/search/messages | Search for messages | +| QUERY | /v1/search/threads | Search for threads | +| QUERY | /v1/search/rooms | Search for rooms | +| ????? | /v1/voip/???? | Anything needed for voip | +| POST | /v1/send-to-device | Send a message directly to a device | +| POST | /v1/keys/upload | Upload/publish encryption keys | +| POST | /v1/keys/upload-cross-keys | Upload/publish cross-signing encryption keys | +| POST | /v1/keys/upload-cross-sigs | Upload/publish cross-signing encryption signatures | +| POST | /v1/keys/query | Get another user's encryption keys | +| POST | /v1/keys/claim | Claim another user's encryption keys | +| PATCH | /v1/key-backup | Add/remove keys from the key backup | +| QUERY | /v1/key-backup | Retrieve keys from the key backup | +| GET | /v1/key-backup/version | Get the latest version | +| POST | /v1/key-backup/version | Create a new version | +| GET | /v1/key-backup/version/{version} | Get a version | +| PUT | /v1/key-backup/version/{version} | Update a version | +| DELETE | /v1/key-backup/version/{version} | Delete a version | + +```rust +// post /v1/rooms/threads +struct GetThreads { + room_ids: Array, + from: Option, + filter: Option, + sort: Option, +} + +struct ThreadFilter { + /// add muted threads + muted: bool, + + /// remove unwatched threads + watching: bool, + + /// remove read threads + unread: bool, +} + +enum ThreadSort { + /// Newest posts first. + #[default] + New, + + /// Oldest posts first. Only used for inbox. + Old, + + /// Sort by the number of +1s (in a time frame) + Votes(Option), + + /// Sort by the number of replies/comments (in a time frame) + Comments(Option), + + /// Sort by the last reply/comment timestamp + Activity, + + /// Sort by the "hot algorithm". By far the most complex. Meant for inbox and large communities. + Hot, +} +``` + +``` +post /v1/rooms/membership +{ + rooms: Array<{ + room: RoomId | RoomAlias, + user_id?: UserId, + membership?: "join" | "leave" | "invite" | "knock" | "ban", + forget?: boolean, // standalone or when membership = leave? + }> +} + +/events +{ + event_id?: EventId, + limit_before?: number, + limit_after?: number, +} + +/relations +{ + event_id: EventId, + rel_type?: string, + event_types?: Array, + depth?: number, // recursion depth? + + // pagination + limit?: number, + after?: string, +} +``` + +```rust +/// The standard "text" type. Used for all user-facing text. +struct Text(Vec); + +struct TextBlock { + /// The content itself. + // NOTE: should this be `any` instead of string? + body: String, + + /// The language of the text (ie. en, en_US, i forget the name for this convention) + lang: Option, + + /// The mime type of the body. May be `text/plain` or `text/html` + // NOTE: maybe should it be `text/jackwagon+html` or `text/html+jackwagon`? + type: Option, +} + +/* +{ + this: "is an example object", + text1: [{ body: "the most basic text block"}], + text2: [ + { body: "now with *formatting*" }, + { body: "now with formatting", type: "text/html" }, + ], + text3: [ + { body: "this may not be exposed in ui, but localization can exist (mostly for bots?)" }, + { body: "(pretend i can speak another language)", lang: "another_LANG" }, + ], +} +*/ +``` + +#### federation + +prefix = `/_jackwagon/federation` + +This is how servers federate with each other. + +| method | path | description | +|--------|------ |------ | +| QUERY | /v1/info | Get public info about this server | +| POST | /v1/send | Send new events (PDUs) | +| POST | /v1/make-join | Ask a server to create a join event, since ours doesn't have room state yet | +| POST | /v1/make-invite | Same as above for inviting other users | +| POST | /v1/make-leave | Same as above for leave (rejecting invites, specifically) | +| POST | /v1/make-knock | Same as above for knock | +| POST | /v1/send-join | Send the join | +| POST | /v1/send-invite | Send the invite | +| POST | /v1/send-leave | Send the leave | +| POST | /v1/send-knock | Send the knock | +| GET | /v1/state/{roomId}/{eventId} | Get a snapshot of room state at a specific event id | +| GET | /v1/state-ids/{roomId}/{eventId} | Like /state, but only the ids instead of full events | +| GET | /v1/event/{roomId}/{eventId} | Get an event | +| GET | /v1/event-auth/{roomId}/{eventId} | Get the full auth chain of an event | +| POST | /v1/backfill/{roomId} | Fetch a range of history | +| POST | /v1/get-missing-events/{roomId} | Fetch some specific events | +| POST | /v1/keys/server | Fetch the server's signing key | +| POST | /v1/keys/query | Ask the server for another server's signing key | +| GET | /v1/hierarchy/{roomId} | Get a space's hierarchy | +| GET | /v1/query/alias/{roomAlias} | Resolve a room alias | +| GET | /v1/query/user/{userId} | Get a user's profile | +| GET | /v1/query/devices/{userId} | Get a user's devices | + +#### media + +prefix = `/_jackwagon/media` + +Specifically for media, which is transferred out of band. I really want +to rework the `blobs` api, since I know it's possible to decentralize +media but still allow streaming it with pubkeys... + +| method | path | description | +|--------|------ |------------- | +| POST | /v1/blobs | Upload a blob | +| GET | /v1/blobs/:id | Download a blob | +| DELETE | /v1/blobs/:id | Delete a blob (only uploader and admins can use) | +| GET | /v1/blobs/:id/thumbnail | Download a blob's server generate thumbnail | +| POST | /v1/url | Generate a url preview | + +#### appservice + +prefix = `/_jackwagon/appservice` + +APIs for appservices. Notably, this api exists *on the appservice*, +not the homeserver and is used for the homeserver to communicate to +the appservice. + +Appservices can pass `?user_id` on any client-server api endpoint to +masquerade as that user. They can also use `?timestamp` to set a custom +origin_server_ts when sending events (though it doesn't rewrite the dag!). + +| method | path | description | +|--------|--------------------------|--------------------------------------------| +| GET | /v1/health | A health check for the application service | +| PUT | /v1/transactions/{txnId} | Receive some events | +| GET | /v1/users/{userId} | Get or create a user's profile | +| GET | /v1/aliases/{roomAlias} | Get or create a room | + +#### admin + +prefix = `/_jackwagon/admin` + +APIs for administrating a server. + +| method | path | description | +|--------|-------------------------------------|-----------------------------------------------------| +| GET | /v1/stats | Get statistics of the server | +| GET | /v1/appservices | Get a list of appservices | +| PUT | /v1/appservices/:id | Register or update an appservice | +| GET | /v1/appservices/:id | Get an appservice's config | +| DELETE | /v1/appservices/:id | Delete an appservice | +| GET | /v1/users | Get a list of all known (or only ?local=true) users | +| GET | /v1/users/{userId} | Get a user | +| PUT | /v1/users/{userId} | Update a user | +| GET | /v1/rooms | Get a list of rooms | +| GET | /v1/rooms/{roomId} | Get a room | +| PUT | /v1/rooms/{roomId} | Update a room | +| POST | /v1/send-notice | Send a server notice | +| POST | /v1/debug/get-pdu/{eventId} | Get a pdu | +| POST | /v1/debug/get-auth/{eventId} | Get a pdu's auth chain | +| POST | /v1/debug/get-extremities/{eventId} | Get a pdu's forward extremities | +| POST | /v1/debug/sign | Sign json | +| POST | /v1/debug/verify | Verify a pdu | + +this should probably be migrated out of REST for bulk deactivation + +```ts +type AppserviceConfig = { + description: string, // a human readable description + as_token: string, // secret token the appservice uses to authenticate itself + hs_token: string, // secret token the homeserver uses to authenticate itself + url: string, // the url the homeserver uses to communicate with the appservice + media: string, // the url the homeserver uses to communicate with the appservice + namespaces: { + users?: Array<{ regex: string }>, + aliases?: Array<{ regex: string }>, + } +} + +// quarantined users cannot dm people among other restrictions +type UserConfig = { + profile: UserProfile, + state: "active" | "quarantined" | "readonly" | "disabled", +}; + +// quarantined rooms cannot be joined by any new users (but existing members can stay), and will not show up in /hierarchy +// exiled rooms cannot be interacted with at all, and any joined users will leave. the server may purge the state. +type RoomConfig = { + state: "active" | "quarantined" | "exiled", +}; +``` + +### primitives + +| name | format | description | +|-|-|-| +|room id|!opaque:domain.tld|A globally unique identifier for a room. The domain part is purely to ensure uniqueness, and is meaningless otherwise.| +|user id|@name:domain.tld|A globally unique identifier for a user.| +|event id|$opaque|A globally unique identifier for an event.| + +### event struct + +| field | type | description | +|-|-|-| +| event_id | event id | a unique identifier for this event | +| room_id? | room id | the room this event originated from* | +| content | any | arbitrary content; in practice, this probably matches the event type | +| sender | user id | the sender of this event | +| state_key? | string | if this is a state event | +| type | string | the type of this event | +| origin_server_ts | number | the timestamp at the server this event originated from | +| unsigned | unsigned data | anything extra the server calculated | + +*only present in some endpoints + +### standard types + +```ts +// Text +// a string is shorthand for a single TextPart of type text/plain +type Text = string | Array; +type TextPart = { + body: string, + lang?: string, + type?: "text/plain" | "text/html" | string, +}; + +type File = { + url: /mxc/, + info: { + alt: Text, + type: number, // mime type + size: number, // in bytes + width?: number, // type = image, video + height?: number, // type = image, video + duration?: number, // type = audio, video + }, +}; + +// Events sent to clients +type ClientEvent = { + event_id: EventId, + room_id?: RoomId, // removed in /sync, since the room id is sent anyway + content: any, + sender: UserId, + state_key?: string, + type: string, + origin_server_ts: number, + unsigned: UnsignedData, +}; + +type ServerEvent = { + auth_events: Array, + content: any, + hashes: any, + origin_server_ts: number, + prev_events: Array, + room_id: RoomId, + sender: UserId, + signatures: any, + state_key?: string, + type: string, + // unsigned: UnsignedData, +}; + +type UnsignedData = { + prev_content?: any, + redacted_because?: ClientEvent, + transaction_id?: string, + relations: { + "m.thread": { + count: number, + participation: "participation", + }, + "m.replace": { + latest_event: ClientEvent, + }, + "m.plus": { + count: number, + }, + "m.annotation": { + annotations: Array, + count: number, + }, + }, + unreads: Unreads, + // per-event account data? + // account_data: any, +}; +``` diff --git a/src/data.md b/src/data.md new file mode 100644 index 0000000..a91924d --- /dev/null +++ b/src/data.md @@ -0,0 +1,11 @@ +# list of data structures + +## events + +`m.message` + +```js +{ + "m.text": "bar" +} +``` diff --git a/src/design.md b/src/design.md new file mode 100644 index 0000000..af2046f --- /dev/null +++ b/src/design.md @@ -0,0 +1,31 @@ +# design + +there are some design choices + +## faq + +### no main instance + +i'm a programmer not a moderator and don't want to deal with that + +yeah it will hurt adoption, but the goal isn't to amass a ton of users + +## ui + +### why thread only + +it strikes a nice balance between async and synchronous. + +i *may* add fully async tree-style reply style rooms. + +fully synchronous chat is not planned. + +## technical + +### the api is not restful + +it uses batching + +### why long polling + +http/3 diff --git a/src/install.md b/src/install.md new file mode 100644 index 0000000..2d111aa --- /dev/null +++ b/src/install.md @@ -0,0 +1,68 @@ +# install + +## docker compose + +it's planned to be modular and composed of multiple parts + +```yaml +version: '3' + +services: + # This is the main jackwagon server + backend: + # image: matrixconduit/matrix-conduit:latest + build: + context: . + args: + CREATED: '2021-03-16T08:18:27Z' + VERSION: '0.1.0' + LOCAL: 'false' + GIT_REF: origin/master + restart: unless-stopped + ports: + - 8448:6167 + volumes: + - db:/var/lib/matrix-conduit/ + environment: + CONDUIT_SERVER_NAME: your.server.name # EDIT THIS + CONDUIT_DATABASE_PATH: /var/lib/matrix-conduit/ + CONDUIT_DATABASE_BACKEND: rocksdb + CONDUIT_PORT: 6167 + CONDUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + CONDUIT_ALLOW_REGISTRATION: 'true' + CONDUIT_ALLOW_FEDERATION: 'true' + CONDUIT_ALLOW_CHECK_FOR_UPDATES: 'true' + CONDUIT_TRUSTED_SERVERS: '["matrix.org"]' + #CONDUIT_MAX_CONCURRENT_REQUESTS: 100 + #CONDUIT_LOG: warn,rocket=off,_=off,sled=off + CONDUIT_ADDRESS: 0.0.0.0 + CONDUIT_CONFIG: '' # Ignore this + + # the main frontend + frontend: + image: vectorim/element-web:latest + restart: unless-stopped + ports: + - 8009:80 + + # handle turn/voip, you can also run this on other servers + coturn: + image: coturn/coturn + restart: unless-stopped + ports: + - 3478:3478 + - 3478:3478/udp + - 5349:5349 + - 5349:5349/udp + - 49152-65535:49152-65535/udp + + # database + postgres: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: example + +volumes: + db: +``` diff --git a/src/syncing.md b/src/syncing.md new file mode 100644 index 0000000..72049d8 --- /dev/null +++ b/src/syncing.md @@ -0,0 +1,65 @@ +# how to sync + +call `POST /v1/sync` (or `QUERY /v1/sync`?) in a loop + +## goals + +1. Be able to incrementally update a local offline copy +2. Be able to fetch only what is needed + +- query and update state, threads, members + +todo: write out + +```ts +type Query = { + state: Array<[string, string]>, // (event_type, state_key) + subscribe: boolean, + threads: { + limit: number, + timeline_limit: number, + }, +}; + +type Request = { + conn_id: string, + pos: string, + delta_token: string, + + lists: Record, + purposes: Array, + spaces: Array, + tombstoned: Array, + }, + } & Query>, + + rooms: Record, + + extensions: { + presence: { + enabled: true, + }, + account_data: { + enabled: true, + types?: Array, + rooms?: Array, + }, + }, +} + +type Reponse = { + lists: Record, + + rooms: Record, + threads: Array, + timeline: Array, + }>, +} +```