mirror of
https://gitlab.com/famedly/conduit.git
synced 2025-04-22 14:10:16 +03:00
Merge branch 'media-refactor' into 'next'
Draft: Media refactor Closes #436, #146, #312, #168, and #421 See merge request famedly/conduit!740
This commit is contained in:
Matthias Ahouansou
commit
12b3a1a211
13 changed files with 2063 additions and 395 deletions
84
Cargo.lock
generated
84
Cargo.lock
generated
|
|
@ -38,6 +38,21 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android-tzdata"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.7"
|
||||
|
|
@ -430,6 +445,20 @@ version = "0.1.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
|
||||
dependencies = [
|
||||
"android-tzdata",
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.8.1"
|
||||
|
|
@ -495,13 +524,16 @@ dependencies = [
|
|||
"axum-server",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"clap",
|
||||
"directories",
|
||||
"figment",
|
||||
"futures-util",
|
||||
"hex",
|
||||
"hickory-resolver",
|
||||
"hmac",
|
||||
"http 1.1.0",
|
||||
"humantime",
|
||||
"hyper 1.3.1",
|
||||
"hyper-util",
|
||||
"image",
|
||||
|
|
@ -528,6 +560,7 @@ dependencies = [
|
|||
"serde_json",
|
||||
"serde_yaml",
|
||||
"sha-1",
|
||||
"sha2",
|
||||
"thiserror 1.0.61",
|
||||
"thread_local",
|
||||
"threadpool",
|
||||
|
|
@ -1045,6 +1078,12 @@ version = "0.3.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hickory-proto"
|
||||
version = "0.24.1"
|
||||
|
|
@ -1187,6 +1226,12 @@ version = "1.0.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f"
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.14.29"
|
||||
|
|
@ -1281,6 +1326,30 @@ dependencies = [
|
|||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.62"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.4.0"
|
||||
|
|
@ -3543,6 +3612,21 @@ version = "0.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
|
|
|
|||
|
|
@ -85,6 +85,9 @@ image = { version = "0.25", default-features = false, features = [
|
|||
"jpeg",
|
||||
"png",
|
||||
] }
|
||||
# Used for creating media filenames
|
||||
hex = "0.4"
|
||||
sha2 = "0.10"
|
||||
# Used to encode server public key
|
||||
base64 = "0.22"
|
||||
# Used when hashing the state
|
||||
|
|
@ -120,6 +123,7 @@ thread_local = "1.1.7"
|
|||
hmac = "0.12.1"
|
||||
sha-1 = "0.10.1"
|
||||
# used for conduit's CLI and admin room command parsing
|
||||
chrono = "0.4"
|
||||
clap = { version = "4.3.0", default-features = false, features = [
|
||||
"derive",
|
||||
"error-context",
|
||||
|
|
@ -128,6 +132,8 @@ clap = { version = "4.3.0", default-features = false, features = [
|
|||
"string",
|
||||
"usage",
|
||||
] }
|
||||
humantime = "2"
|
||||
|
||||
futures-util = { version = "0.3.28", default-features = false }
|
||||
# Used for reading the configuration from conduit.toml & environment variables
|
||||
figment = { version = "0.10.8", features = ["env", "toml"] }
|
||||
|
|
|
|||
|
|
@ -57,9 +57,47 @@ The `global` section contains the following fields:
|
|||
| `turn_uris` | `array` | The TURN URIs | `[]` |
|
||||
| `turn_secret` | `string` | The TURN secret | `""` |
|
||||
| `turn_ttl` | `integer` | The TURN TTL in seconds | `86400` |
|
||||
| `media` | `table` | See the [media configuration](#media) | See the [media configuration](#media) |
|
||||
| `emergency_password` | `string` | Set a password to login as the `conduit` user in case of emergency | N/A |
|
||||
| `well_known` | `table` | Used for [delegation](delegation.md) | See [delegation](delegation.md) |
|
||||
|
||||
### Media
|
||||
The `media` table is used to configure how media is stored and where. Currently, there is only one available
|
||||
backend, that being `filesystem`. The backend can be set using the `backend` field. Example:
|
||||
```toml
|
||||
[global.media]
|
||||
backend = "filesystem" # the default backend
|
||||
```
|
||||
|
||||
#### Filesystem backend
|
||||
The filesystem backend has the following fields:
|
||||
- `path`: The base directory where all the media files will be stored (defaults to
|
||||
`${database_path}/media`)
|
||||
- `directory_structure`: This is a table, used to configure how files are to be distributed within
|
||||
the media directory. It has the following fields:
|
||||
- `depth`: The number sub-directories that should be created for files (default: `1`)
|
||||
- `length`: How long the name of these sub-directories should be (default: `4`)
|
||||
For example, a file may regularly have the name `98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4`
|
||||
(The SHA256 digest of the file's content). If `depth` and `length` were both set to `2`, this file would be stored
|
||||
at `${path}/98/ea/6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4`. If you want to instead have all
|
||||
media files in the base directory with no sub-directories, just set `directory_structure` to be empty, as follows:
|
||||
```toml
|
||||
[global.media]
|
||||
backend = "filesystem"
|
||||
|
||||
[global.media.directory_structure]
|
||||
```
|
||||
|
||||
##### Example:
|
||||
```toml
|
||||
[global.media]
|
||||
backend = "filesystem"
|
||||
path = "/srv/matrix-media"
|
||||
|
||||
[global.media.directory_structure]
|
||||
depth = 4
|
||||
length = 2
|
||||
```
|
||||
|
||||
### TLS
|
||||
The `tls` table contains the following fields:
|
||||
|
|
|
|||
|
|
@ -54,33 +54,34 @@ pub async fn get_media_config_auth_route(
|
|||
pub async fn create_content_route(
|
||||
body: Ruma<create_content::v3::Request>,
|
||||
) -> Result<create_content::v3::Response> {
|
||||
let mxc = format!(
|
||||
"mxc://{}/{}",
|
||||
services().globals.server_name(),
|
||||
utils::random_string(MXC_LENGTH)
|
||||
);
|
||||
let create_content::v3::Request {
|
||||
filename,
|
||||
content_type,
|
||||
file,
|
||||
..
|
||||
} = body.body;
|
||||
|
||||
let media_id = utils::random_string(MXC_LENGTH);
|
||||
|
||||
services()
|
||||
.media
|
||||
.create(
|
||||
mxc.clone(),
|
||||
Some(
|
||||
ContentDisposition::new(ContentDispositionType::Inline)
|
||||
.with_filename(body.filename.clone()),
|
||||
),
|
||||
body.content_type.as_deref(),
|
||||
&body.file,
|
||||
services().globals.server_name(),
|
||||
&media_id,
|
||||
filename.as_deref(),
|
||||
content_type.as_deref(),
|
||||
&file,
|
||||
body.sender_user.as_deref(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(create_content::v3::Response {
|
||||
content_uri: mxc.into(),
|
||||
content_uri: (format!("mxc://{}/{}", services().globals.server_name(), media_id)).into(),
|
||||
blurhash: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn get_remote_content(
|
||||
mxc: &str,
|
||||
server_name: &ServerName,
|
||||
media_id: String,
|
||||
) -> Result<get_content::v1::Response, Error> {
|
||||
|
|
@ -120,7 +121,7 @@ pub async fn get_remote_content(
|
|||
server_name,
|
||||
media::get_content::v3::Request {
|
||||
server_name: server_name.to_owned(),
|
||||
media_id,
|
||||
media_id: media_id.clone(),
|
||||
timeout_ms: Duration::from_secs(20),
|
||||
allow_remote: false,
|
||||
allow_redirect: true,
|
||||
|
|
@ -140,10 +141,15 @@ pub async fn get_remote_content(
|
|||
services()
|
||||
.media
|
||||
.create(
|
||||
mxc.to_owned(),
|
||||
content_response.content_disposition.clone(),
|
||||
server_name,
|
||||
&media_id,
|
||||
content_response
|
||||
.content_disposition
|
||||
.as_ref()
|
||||
.and_then(|cd| cd.filename.as_deref()),
|
||||
content_response.content_type.as_deref(),
|
||||
&content_response.file,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -162,7 +168,13 @@ pub async fn get_content_route(
|
|||
file,
|
||||
content_disposition,
|
||||
content_type,
|
||||
} = get_content(&body.server_name, body.media_id.clone(), body.allow_remote).await?;
|
||||
} = get_content(
|
||||
&body.server_name,
|
||||
body.media_id.clone(),
|
||||
body.allow_remote,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(media::get_content::v3::Response {
|
||||
file,
|
||||
|
|
@ -178,21 +190,25 @@ pub async fn get_content_route(
|
|||
pub async fn get_content_auth_route(
|
||||
body: Ruma<get_content::v1::Request>,
|
||||
) -> Result<get_content::v1::Response> {
|
||||
get_content(&body.server_name, body.media_id.clone(), true).await
|
||||
get_content(&body.server_name, body.media_id.clone(), true, true).await
|
||||
}
|
||||
|
||||
async fn get_content(
|
||||
server_name: &ServerName,
|
||||
media_id: String,
|
||||
allow_remote: bool,
|
||||
authenticated: bool,
|
||||
) -> Result<get_content::v1::Response, Error> {
|
||||
let mxc = format!("mxc://{}/{}", server_name, media_id);
|
||||
services().media.check_blocked(server_name, &media_id)?;
|
||||
|
||||
if let Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file,
|
||||
})) = services().media.get(mxc.clone()).await
|
||||
})) = services()
|
||||
.media
|
||||
.get(server_name, &media_id, authenticated)
|
||||
.await
|
||||
{
|
||||
Ok(get_content::v1::Response {
|
||||
file,
|
||||
|
|
@ -200,8 +216,7 @@ async fn get_content(
|
|||
content_disposition: Some(content_disposition),
|
||||
})
|
||||
} else if server_name != services().globals.server_name() && allow_remote {
|
||||
let remote_content_response =
|
||||
get_remote_content(&mxc, server_name, media_id.clone()).await?;
|
||||
let remote_content_response = get_remote_content(server_name, media_id.clone()).await?;
|
||||
|
||||
Ok(get_content::v1::Response {
|
||||
content_disposition: remote_content_response.content_disposition,
|
||||
|
|
@ -230,6 +245,7 @@ pub async fn get_content_as_filename_route(
|
|||
body.media_id.clone(),
|
||||
body.filename.clone(),
|
||||
body.allow_remote,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -252,6 +268,7 @@ pub async fn get_content_as_filename_auth_route(
|
|||
body.media_id.clone(),
|
||||
body.filename.clone(),
|
||||
true,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
|
@ -261,12 +278,16 @@ async fn get_content_as_filename(
|
|||
media_id: String,
|
||||
filename: String,
|
||||
allow_remote: bool,
|
||||
authenticated: bool,
|
||||
) -> Result<get_content_as_filename::v1::Response, Error> {
|
||||
let mxc = format!("mxc://{}/{}", server_name, media_id);
|
||||
services().media.check_blocked(server_name, &media_id)?;
|
||||
|
||||
if let Ok(Some(FileMeta {
|
||||
file, content_type, ..
|
||||
})) = services().media.get(mxc.clone()).await
|
||||
})) = services()
|
||||
.media
|
||||
.get(server_name, &media_id, authenticated)
|
||||
.await
|
||||
{
|
||||
Ok(get_content_as_filename::v1::Response {
|
||||
file,
|
||||
|
|
@ -277,8 +298,7 @@ async fn get_content_as_filename(
|
|||
),
|
||||
})
|
||||
} else if server_name != services().globals.server_name() && allow_remote {
|
||||
let remote_content_response =
|
||||
get_remote_content(&mxc, server_name, media_id.clone()).await?;
|
||||
let remote_content_response = get_remote_content(server_name, media_id.clone()).await?;
|
||||
|
||||
Ok(get_content_as_filename::v1::Response {
|
||||
content_disposition: Some(
|
||||
|
|
@ -313,6 +333,7 @@ pub async fn get_content_thumbnail_route(
|
|||
body.method.clone(),
|
||||
body.animated,
|
||||
body.allow_remote,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -338,10 +359,12 @@ pub async fn get_content_thumbnail_auth_route(
|
|||
body.method.clone(),
|
||||
body.animated,
|
||||
true,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn get_content_thumbnail(
|
||||
server_name: &ServerName,
|
||||
media_id: String,
|
||||
|
|
@ -350,8 +373,9 @@ async fn get_content_thumbnail(
|
|||
method: Option<Method>,
|
||||
animated: Option<bool>,
|
||||
allow_remote: bool,
|
||||
authenticated: bool,
|
||||
) -> Result<get_content_thumbnail::v1::Response, Error> {
|
||||
let mxc = format!("mxc://{}/{}", server_name, media_id);
|
||||
services().media.check_blocked(server_name, &media_id)?;
|
||||
|
||||
if let Some(FileMeta {
|
||||
file,
|
||||
|
|
@ -360,13 +384,15 @@ async fn get_content_thumbnail(
|
|||
}) = services()
|
||||
.media
|
||||
.get_thumbnail(
|
||||
mxc.clone(),
|
||||
server_name,
|
||||
&media_id,
|
||||
width
|
||||
.try_into()
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
|
||||
height
|
||||
.try_into()
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Height is invalid."))?,
|
||||
authenticated,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
|
@ -452,7 +478,12 @@ async fn get_content_thumbnail(
|
|||
services()
|
||||
.media
|
||||
.upload_thumbnail(
|
||||
mxc,
|
||||
server_name,
|
||||
&media_id,
|
||||
thumbnail_response
|
||||
.content_disposition
|
||||
.as_ref()
|
||||
.and_then(|cd| cd.filename.as_deref()),
|
||||
thumbnail_response.content_type.as_deref(),
|
||||
width.try_into().expect("all UInts are valid u32s"),
|
||||
height.try_into().expect("all UInts are valid u32s"),
|
||||
|
|
|
|||
|
|
@ -2215,23 +2215,24 @@ pub async fn create_invite_route(
|
|||
})
|
||||
}
|
||||
|
||||
/// # `GET /_matrix/federation/v1/media/download/{serverName}/{mediaId}`
|
||||
/// # `GET /_matrix/federation/v1/media/download/{mediaId}`
|
||||
///
|
||||
/// Load media from our server.
|
||||
pub async fn get_content_route(
|
||||
body: Ruma<get_content::v1::Request>,
|
||||
) -> Result<get_content::v1::Response> {
|
||||
let mxc = format!(
|
||||
"mxc://{}/{}",
|
||||
services().globals.server_name(),
|
||||
body.media_id
|
||||
);
|
||||
services()
|
||||
.media
|
||||
.check_blocked(services().globals.server_name(), &body.media_id)?;
|
||||
|
||||
if let Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file,
|
||||
}) = services().media.get(mxc.clone()).await?
|
||||
}) = services()
|
||||
.media
|
||||
.get(services().globals.server_name(), &body.media_id, true)
|
||||
.await?
|
||||
{
|
||||
Ok(get_content::v1::Response::new(
|
||||
ContentMetadata::new(),
|
||||
|
|
@ -2246,17 +2247,15 @@ pub async fn get_content_route(
|
|||
}
|
||||
}
|
||||
|
||||
/// # `GET /_matrix/federation/v1/media/thumbnail/{serverName}/{mediaId}`
|
||||
/// # `GET /_matrix/federation/v1/media/thumbnail/{mediaId}`
|
||||
///
|
||||
/// Load media thumbnail from our server or over federation.
|
||||
pub async fn get_content_thumbnail_route(
|
||||
body: Ruma<get_content_thumbnail::v1::Request>,
|
||||
) -> Result<get_content_thumbnail::v1::Response> {
|
||||
let mxc = format!(
|
||||
"mxc://{}/{}",
|
||||
services().globals.server_name(),
|
||||
body.media_id
|
||||
);
|
||||
services()
|
||||
.media
|
||||
.check_blocked(services().globals.server_name(), &body.media_id)?;
|
||||
|
||||
let Some(FileMeta {
|
||||
file,
|
||||
|
|
@ -2265,13 +2264,15 @@ pub async fn get_content_thumbnail_route(
|
|||
}) = services()
|
||||
.media
|
||||
.get_thumbnail(
|
||||
mxc.clone(),
|
||||
services().globals.server_name(),
|
||||
&body.media_id,
|
||||
body.width
|
||||
.try_into()
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
|
||||
body.height
|
||||
.try_into()
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
|
||||
true,
|
||||
)
|
||||
.await?
|
||||
else {
|
||||
|
|
@ -2281,7 +2282,9 @@ pub async fn get_content_thumbnail_route(
|
|||
services()
|
||||
.media
|
||||
.upload_thumbnail(
|
||||
mxc,
|
||||
services().globals.server_name(),
|
||||
&body.media_id,
|
||||
content_disposition.filename.as_deref(),
|
||||
content_type.as_deref(),
|
||||
body.width.try_into().expect("all UInts are valid u32s"),
|
||||
body.height.try_into().expect("all UInts are valid u32s"),
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ use std::{
|
|||
collections::BTreeMap,
|
||||
fmt,
|
||||
net::{IpAddr, Ipv4Addr},
|
||||
num::NonZeroU8,
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use ruma::{OwnedServerName, RoomVersionId};
|
||||
|
|
@ -9,12 +11,15 @@ use serde::{de::IgnoredAny, Deserialize};
|
|||
use tracing::warn;
|
||||
use url::Url;
|
||||
|
||||
mod proxy;
|
||||
use crate::Error;
|
||||
|
||||
mod proxy;
|
||||
use self::proxy::ProxyConfig;
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct Config {
|
||||
const SHA256_HEX_LENGTH: u8 = 64;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct IncompleteConfig {
|
||||
#[serde(default = "default_address")]
|
||||
pub address: IpAddr,
|
||||
#[serde(default = "default_port")]
|
||||
|
|
@ -60,7 +65,7 @@ pub struct Config {
|
|||
#[serde(default = "default_default_room_version")]
|
||||
pub default_room_version: RoomVersionId,
|
||||
#[serde(default)]
|
||||
pub well_known: WellKnownConfig,
|
||||
pub well_known: IncompleteWellKnownConfig,
|
||||
#[serde(default = "false_fn")]
|
||||
pub allow_jaeger: bool,
|
||||
#[serde(default = "false_fn")]
|
||||
|
|
@ -81,12 +86,200 @@ pub struct Config {
|
|||
|
||||
pub turn: Option<TurnConfig>,
|
||||
|
||||
#[serde(default)]
|
||||
pub media: IncompleteMediaConfig,
|
||||
|
||||
pub emergency_password: Option<String>,
|
||||
|
||||
#[serde(flatten)]
|
||||
pub catchall: BTreeMap<String, IgnoredAny>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Clone, Debug)]
|
||||
#[serde(from = "IncompleteConfig")]
|
||||
pub struct Config {
|
||||
pub address: IpAddr,
|
||||
pub port: u16,
|
||||
pub tls: Option<TlsConfig>,
|
||||
|
||||
pub server_name: OwnedServerName,
|
||||
pub database_backend: String,
|
||||
pub database_path: String,
|
||||
pub db_cache_capacity_mb: f64,
|
||||
pub enable_lightning_bolt: bool,
|
||||
pub allow_check_for_updates: bool,
|
||||
pub conduit_cache_capacity_modifier: f64,
|
||||
pub rocksdb_max_open_files: i32,
|
||||
pub pdu_cache_capacity: u32,
|
||||
pub cleanup_second_interval: u32,
|
||||
pub max_request_size: u32,
|
||||
pub max_concurrent_requests: u16,
|
||||
pub max_fetch_prev_events: u16,
|
||||
pub allow_registration: bool,
|
||||
pub registration_token: Option<String>,
|
||||
pub openid_token_ttl: u64,
|
||||
pub allow_encryption: bool,
|
||||
pub allow_federation: bool,
|
||||
pub allow_room_creation: bool,
|
||||
pub allow_unstable_room_versions: bool,
|
||||
pub default_room_version: RoomVersionId,
|
||||
pub well_known: WellKnownConfig,
|
||||
pub allow_jaeger: bool,
|
||||
pub tracing_flame: bool,
|
||||
pub proxy: ProxyConfig,
|
||||
pub jwt_secret: Option<String>,
|
||||
pub trusted_servers: Vec<OwnedServerName>,
|
||||
pub log: String,
|
||||
|
||||
pub turn: Option<TurnConfig>,
|
||||
|
||||
pub media: MediaConfig,
|
||||
|
||||
pub emergency_password: Option<String>,
|
||||
|
||||
pub catchall: BTreeMap<String, IgnoredAny>,
|
||||
}
|
||||
|
||||
impl From<IncompleteConfig> for Config {
|
||||
fn from(val: IncompleteConfig) -> Self {
|
||||
let IncompleteConfig {
|
||||
address,
|
||||
port,
|
||||
tls,
|
||||
server_name,
|
||||
database_backend,
|
||||
database_path,
|
||||
db_cache_capacity_mb,
|
||||
enable_lightning_bolt,
|
||||
allow_check_for_updates,
|
||||
conduit_cache_capacity_modifier,
|
||||
rocksdb_max_open_files,
|
||||
pdu_cache_capacity,
|
||||
cleanup_second_interval,
|
||||
max_request_size,
|
||||
max_concurrent_requests,
|
||||
max_fetch_prev_events,
|
||||
allow_registration,
|
||||
registration_token,
|
||||
openid_token_ttl,
|
||||
allow_encryption,
|
||||
allow_federation,
|
||||
allow_room_creation,
|
||||
allow_unstable_room_versions,
|
||||
default_room_version,
|
||||
well_known,
|
||||
allow_jaeger,
|
||||
tracing_flame,
|
||||
proxy,
|
||||
jwt_secret,
|
||||
trusted_servers,
|
||||
log,
|
||||
turn_username,
|
||||
turn_password,
|
||||
turn_uris,
|
||||
turn_secret,
|
||||
turn_ttl,
|
||||
turn,
|
||||
media,
|
||||
emergency_password,
|
||||
catchall,
|
||||
} = val;
|
||||
|
||||
let turn = turn.or_else(|| {
|
||||
let auth = if let Some(secret) = turn_secret {
|
||||
TurnAuth::Secret { secret }
|
||||
} else if let (Some(username), Some(password)) = (turn_username, turn_password) {
|
||||
TurnAuth::UserPass { username, password }
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
|
||||
if let (Some(uris), ttl) = (turn_uris, turn_ttl) {
|
||||
Some(TurnConfig { uris, ttl, auth })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
let well_known_client = well_known
|
||||
.client
|
||||
.map(String::from)
|
||||
.unwrap_or_else(|| format!("https://{server_name}"));
|
||||
|
||||
let well_known_server = well_known.server.unwrap_or_else(|| {
|
||||
if server_name.port().is_some() {
|
||||
server_name.clone()
|
||||
} else {
|
||||
format!("{}:443", server_name.host())
|
||||
.try_into()
|
||||
.expect("Host from valid hostname + :443 must be valid")
|
||||
}
|
||||
});
|
||||
|
||||
let well_known = WellKnownConfig {
|
||||
client: well_known_client,
|
||||
server: well_known_server,
|
||||
};
|
||||
|
||||
let media = match media {
|
||||
IncompleteMediaConfig::FileSystem {
|
||||
path,
|
||||
directory_structure,
|
||||
} => MediaConfig::FileSystem {
|
||||
path: path.unwrap_or_else(|| {
|
||||
// We do this as we don't know if the path has a trailing slash, or even if the
|
||||
// path separator is a forward or backward slash
|
||||
[&database_path, "media"]
|
||||
.iter()
|
||||
.collect::<PathBuf>()
|
||||
.into_os_string()
|
||||
.into_string()
|
||||
.expect("Both inputs are valid UTF-8")
|
||||
}),
|
||||
directory_structure,
|
||||
},
|
||||
};
|
||||
|
||||
Config {
|
||||
address,
|
||||
port,
|
||||
tls,
|
||||
server_name,
|
||||
database_backend,
|
||||
database_path,
|
||||
db_cache_capacity_mb,
|
||||
enable_lightning_bolt,
|
||||
allow_check_for_updates,
|
||||
conduit_cache_capacity_modifier,
|
||||
rocksdb_max_open_files,
|
||||
pdu_cache_capacity,
|
||||
cleanup_second_interval,
|
||||
max_request_size,
|
||||
max_concurrent_requests,
|
||||
max_fetch_prev_events,
|
||||
allow_registration,
|
||||
registration_token,
|
||||
openid_token_ttl,
|
||||
allow_encryption,
|
||||
allow_federation,
|
||||
allow_room_creation,
|
||||
allow_unstable_room_versions,
|
||||
default_room_version,
|
||||
well_known,
|
||||
allow_jaeger,
|
||||
tracing_flame,
|
||||
proxy,
|
||||
jwt_secret,
|
||||
trusted_servers,
|
||||
log,
|
||||
turn,
|
||||
media,
|
||||
emergency_password,
|
||||
catchall,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct TlsConfig {
|
||||
pub certs: String,
|
||||
|
|
@ -110,11 +303,101 @@ pub enum TurnAuth {
|
|||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Default)]
|
||||
pub struct WellKnownConfig {
|
||||
pub struct IncompleteWellKnownConfig {
|
||||
// We use URL here so that the user gets an error if the config isn't a valid url
|
||||
pub client: Option<Url>,
|
||||
pub server: Option<OwnedServerName>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct WellKnownConfig {
|
||||
// We use String here as there is no point converting our manually constructed String into a
|
||||
// URL, just for it to be converted back into a &str
|
||||
pub client: String,
|
||||
pub server: OwnedServerName,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "backend", rename_all = "lowercase")]
|
||||
pub enum IncompleteMediaConfig {
|
||||
FileSystem {
|
||||
path: Option<String>,
|
||||
#[serde(default)]
|
||||
directory_structure: DirectoryStructure,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for IncompleteMediaConfig {
|
||||
fn default() -> Self {
|
||||
Self::FileSystem {
|
||||
path: None,
|
||||
directory_structure: DirectoryStructure::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum MediaConfig {
|
||||
FileSystem {
|
||||
path: String,
|
||||
directory_structure: DirectoryStructure,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
// See https://github.com/serde-rs/serde/issues/642#issuecomment-525432907
|
||||
#[serde(try_from = "ShadowDirectoryStructure", untagged)]
|
||||
pub enum DirectoryStructure {
|
||||
// We do this enum instead of Option<DirectoryStructure>, so that we can have the structure be
|
||||
// deep by default, while still providing a away for it to be flat (by creating an empty table)
|
||||
//
|
||||
// e.g.:
|
||||
// ```toml
|
||||
// [global.media.directory_structure]
|
||||
// ```
|
||||
Flat,
|
||||
Deep { length: NonZeroU8, depth: NonZeroU8 },
|
||||
}
|
||||
|
||||
impl Default for DirectoryStructure {
|
||||
fn default() -> Self {
|
||||
Self::Deep {
|
||||
length: NonZeroU8::new(4).expect("4 is not 0"),
|
||||
depth: NonZeroU8::new(1).expect("1 is not 0"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum ShadowDirectoryStructure {
|
||||
Flat,
|
||||
Deep { length: NonZeroU8, depth: NonZeroU8 },
|
||||
}
|
||||
|
||||
impl TryFrom<ShadowDirectoryStructure> for DirectoryStructure {
|
||||
type Error = Error;
|
||||
|
||||
fn try_from(value: ShadowDirectoryStructure) -> Result<Self, Self::Error> {
|
||||
match value {
|
||||
ShadowDirectoryStructure::Flat => Ok(Self::Flat),
|
||||
ShadowDirectoryStructure::Deep { length, depth } => {
|
||||
if length
|
||||
.get()
|
||||
.checked_mul(depth.get())
|
||||
.map(|product| product < SHA256_HEX_LENGTH)
|
||||
// If an overflow occurs, it definitely isn't less than SHA256_HEX_LENGTH
|
||||
.unwrap_or(false)
|
||||
{
|
||||
Ok(Self::Deep { length, depth })
|
||||
} else {
|
||||
Err(Error::bad_config("The media directory structure depth multiplied by the depth is equal to or greater than a sha256 hex hash, please reduce at least one of the two so that their product is less than 64"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const DEPRECATED_KEYS: &[&str] = &[
|
||||
"cache_capacity",
|
||||
"turn_username",
|
||||
|
|
@ -142,61 +425,9 @@ impl Config {
|
|||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn well_known_client(&self) -> String {
|
||||
if let Some(url) = &self.well_known.client {
|
||||
url.to_string()
|
||||
} else {
|
||||
format!("https://{}", self.server_name)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn well_known_server(&self) -> OwnedServerName {
|
||||
match &self.well_known.server {
|
||||
Some(server_name) => server_name.to_owned(),
|
||||
None => {
|
||||
if self.server_name.port().is_some() {
|
||||
self.server_name.to_owned()
|
||||
} else {
|
||||
format!("{}:443", self.server_name.host())
|
||||
.try_into()
|
||||
.expect("Host from valid hostname + :443 must be valid")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn turn(&self) -> Option<TurnConfig> {
|
||||
if self.turn.is_some() {
|
||||
self.turn.clone()
|
||||
} else if let Some(uris) = self.turn_uris.clone() {
|
||||
if let Some(secret) = self.turn_secret.clone() {
|
||||
Some(TurnConfig {
|
||||
uris,
|
||||
ttl: self.turn_ttl,
|
||||
auth: TurnAuth::Secret { secret },
|
||||
})
|
||||
} else if let (Some(username), Some(password)) =
|
||||
(self.turn_username.clone(), self.turn_password.clone())
|
||||
{
|
||||
Some(TurnConfig {
|
||||
uris,
|
||||
ttl: self.turn_ttl,
|
||||
auth: TurnAuth::UserPass { username, password },
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Config {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
// Prepare a list of config values to show
|
||||
let well_known_server = self.well_known_server();
|
||||
let lines = [
|
||||
("Server name", self.server_name.host()),
|
||||
("Database backend", &self.database_backend),
|
||||
|
|
@ -247,7 +478,7 @@ impl fmt::Display for Config {
|
|||
&lst.join(", ")
|
||||
}),
|
||||
("TURN URIs", {
|
||||
if let Some(turn) = self.turn() {
|
||||
if let Some(turn) = &self.turn {
|
||||
let mut lst = vec![];
|
||||
for item in turn.uris.iter().cloned().enumerate() {
|
||||
let (_, uri): (usize, String) = item;
|
||||
|
|
@ -258,8 +489,8 @@ impl fmt::Display for Config {
|
|||
"unset"
|
||||
}
|
||||
}),
|
||||
("Well-known server name", well_known_server.as_str()),
|
||||
("Well-known client URL", &self.well_known_client()),
|
||||
("Well-known server name", self.well_known.server.as_str()),
|
||||
("Well-known client URL", &self.well_known.client),
|
||||
];
|
||||
|
||||
let mut msg: String = "Active config values:\n\n".to_owned();
|
||||
|
|
|
|||
|
|
@ -1,71 +1,623 @@
|
|||
use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition};
|
||||
use ruma::{api::client::error::ErrorKind, OwnedServerName, ServerName, UserId};
|
||||
use sha2::digest::Output;
|
||||
use sha2::Sha256;
|
||||
|
||||
use crate::{database::KeyValueDatabase, service, utils, Error, Result};
|
||||
use crate::{
|
||||
database::KeyValueDatabase,
|
||||
service::{self, media::BlockedMediaInfo},
|
||||
utils, Error, Result,
|
||||
};
|
||||
|
||||
impl service::media::Data for KeyValueDatabase {
|
||||
fn create_file_metadata(
|
||||
&self,
|
||||
mxc: String,
|
||||
width: u32,
|
||||
height: u32,
|
||||
content_disposition: &ContentDisposition,
|
||||
sha256_digest: Output<Sha256>,
|
||||
file_size: u64,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<Vec<u8>> {
|
||||
let mut key = mxc.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(&width.to_be_bytes());
|
||||
key.extend_from_slice(&height.to_be_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(content_disposition.to_string().as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(
|
||||
content_type
|
||||
.as_ref()
|
||||
.map(|c| c.as_bytes())
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
user_id: Option<&UserId>,
|
||||
) -> Result<()> {
|
||||
let now = utils::secs_since_unix_epoch().to_be_bytes();
|
||||
|
||||
self.mediaid_file.insert(&key, &[])?;
|
||||
let key = sha256_digest.to_vec();
|
||||
|
||||
Ok(key)
|
||||
let mut value = file_size.to_be_bytes().to_vec();
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(&now);
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(&now);
|
||||
|
||||
self.filehash_metadata.insert(&key, &value)?;
|
||||
|
||||
let mut value = key;
|
||||
|
||||
let mut key = servername.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
value.extend_from_slice(filename.map(|f| f.as_bytes()).unwrap_or_default());
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(content_type.map(|f| f.as_bytes()).unwrap_or_default());
|
||||
|
||||
self.servernamemediaid_metadata.insert(&key, &value)?;
|
||||
|
||||
let mut key = sha256_digest.to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(servername.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.filehash_servername_mediaid.insert(&key, &[])?;
|
||||
|
||||
if let Some(user_id) = user_id {
|
||||
let mut key = servername.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(user_id.localpart().as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.servername_userlocalpart_mediaid.insert(&key, &[])?;
|
||||
|
||||
let mut key = servername.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.servernamemediaid_userlocalpart
|
||||
.insert(&key, user_id.as_bytes())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn search_file_metadata(
|
||||
&self,
|
||||
mxc: String,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<(ContentDisposition, Option<String>, Vec<u8>)> {
|
||||
let mut prefix = mxc.as_bytes().to_vec();
|
||||
prefix.push(0xff);
|
||||
prefix.extend_from_slice(&width.to_be_bytes());
|
||||
prefix.extend_from_slice(&height.to_be_bytes());
|
||||
prefix.push(0xff);
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
) -> Result<(String, Option<String>, Option<String>, bool)> {
|
||||
let mut key = servername.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
let (key, _) = self
|
||||
.mediaid_file
|
||||
.scan_prefix(prefix)
|
||||
.next()
|
||||
let value = self
|
||||
.servernamemediaid_metadata
|
||||
.get(&key)?
|
||||
.ok_or(Error::BadRequest(ErrorKind::NotFound, "Media not found"))?;
|
||||
|
||||
let mut parts = key.rsplit(|&b| b == 0xff);
|
||||
let mut parts = value.split(|&b| b == 0xff);
|
||||
|
||||
let sha256 = hex::encode(parts.next().ok_or_else(|| {
|
||||
Error::bad_database("Invalid format for metadata in servernamemediaid_metadata")
|
||||
})?);
|
||||
|
||||
let filename = parts
|
||||
.next()
|
||||
.map(|bytes| {
|
||||
utils::string_from_bytes(bytes).map_err(|_| {
|
||||
Error::bad_database("filename in servernamemediaid_metadata is invalid unicode")
|
||||
})
|
||||
})
|
||||
.transpose()?
|
||||
.and_then(|s| (!s.is_empty()).then_some(s));
|
||||
|
||||
let content_type = parts
|
||||
.next()
|
||||
.map(|bytes| {
|
||||
utils::string_from_bytes(bytes).map_err(|_| {
|
||||
Error::bad_database("Content type in mediaid_file is invalid unicode.")
|
||||
Error::bad_database(
|
||||
"content type in servernamemediaid_metadata is invalid unicode",
|
||||
)
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
.transpose()?
|
||||
.and_then(|s| (!s.is_empty()).then_some(s));
|
||||
|
||||
let content_disposition_bytes = parts
|
||||
let unauthenticated_access_permitted = parts.next().is_some_and(|v| v.is_empty());
|
||||
|
||||
Ok((
|
||||
sha256,
|
||||
filename,
|
||||
content_type,
|
||||
unauthenticated_access_permitted,
|
||||
))
|
||||
}
|
||||
|
||||
fn create_thumbnail_metadata(
|
||||
&self,
|
||||
sha256_digest: Output<Sha256>,
|
||||
file_size: u64,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<()> {
|
||||
let now = utils::secs_since_unix_epoch().to_be_bytes();
|
||||
|
||||
let key = sha256_digest.to_vec();
|
||||
|
||||
let mut value = file_size.to_be_bytes().to_vec();
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(&now);
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(&now);
|
||||
|
||||
self.filehash_metadata.insert(&key, &value)?;
|
||||
|
||||
let mut value = key;
|
||||
|
||||
let mut key = servername.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(&width.to_be_bytes());
|
||||
key.extend_from_slice(&height.to_be_bytes());
|
||||
|
||||
value.extend_from_slice(filename.map(|f| f.as_bytes()).unwrap_or_default());
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(content_type.map(|f| f.as_bytes()).unwrap_or_default());
|
||||
|
||||
self.thumbnailid_metadata.insert(&key, &value)?;
|
||||
|
||||
let mut key = sha256_digest.to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(servername.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(&width.to_be_bytes());
|
||||
key.extend_from_slice(&height.to_be_bytes());
|
||||
|
||||
self.filehash_thumbnailid.insert(&key, &[])
|
||||
}
|
||||
|
||||
fn search_thumbnail_metadata(
|
||||
&self,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<(String, Option<String>, Option<String>, bool)> {
|
||||
let mut key = servername.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(&width.to_be_bytes());
|
||||
key.extend_from_slice(&height.to_be_bytes());
|
||||
|
||||
//TODO: Don't just copy-paste from above functions
|
||||
let value = self
|
||||
.thumbnailid_metadata
|
||||
.get(&key)?
|
||||
.ok_or(Error::BadRequest(ErrorKind::NotFound, "Media not found"))?;
|
||||
|
||||
let mut parts = value.split(|&b| b == 0xff);
|
||||
|
||||
let sha256 = hex::encode(parts.next().ok_or_else(|| {
|
||||
Error::bad_database("Invalid format for metadata in thumbnailid_metadata")
|
||||
})?);
|
||||
|
||||
let filename = parts
|
||||
.next()
|
||||
.ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?;
|
||||
.map(|bytes| {
|
||||
utils::string_from_bytes(bytes).map_err(|_| {
|
||||
Error::bad_database("filename in thumbnailid_metadata is invalid unicode")
|
||||
})
|
||||
})
|
||||
.transpose()?
|
||||
.and_then(|s| (!s.is_empty()).then_some(s));
|
||||
|
||||
let content_disposition = content_disposition_bytes.try_into().unwrap_or_else(|_| {
|
||||
ContentDisposition::new(ruma::http_headers::ContentDispositionType::Inline)
|
||||
});
|
||||
Ok((content_disposition, content_type, key))
|
||||
let content_type = parts
|
||||
.next()
|
||||
.map(|bytes| {
|
||||
utils::string_from_bytes(bytes).map_err(|_| {
|
||||
Error::bad_database("content type in thumbnailid_metadata is invalid unicode")
|
||||
})
|
||||
})
|
||||
.transpose()?
|
||||
.and_then(|s| (!s.is_empty()).then_some(s));
|
||||
|
||||
let unauthenticated_access_permitted = parts.next().is_some_and(|v| v.is_empty());
|
||||
|
||||
Ok((
|
||||
sha256,
|
||||
filename,
|
||||
content_type,
|
||||
unauthenticated_access_permitted,
|
||||
))
|
||||
}
|
||||
|
||||
fn purge_and_get_hashes(&self, media: &[(OwnedServerName, String)]) -> Vec<Result<String>> {
|
||||
media
|
||||
.iter()
|
||||
.map(|(server_name, media_id)| {
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
if let Some(localpart) = self.servernamemediaid_userlocalpart.get(&key)? {
|
||||
self.servernamemediaid_userlocalpart.remove(&key)?;
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(&localpart);
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.servername_userlocalpart_mediaid.remove(&key)?;
|
||||
}
|
||||
|
||||
let value = self.servernamemediaid_metadata.get(&key)?.unwrap();
|
||||
self.servernamemediaid_metadata.remove(&key)?;
|
||||
|
||||
let mut parts = value.split(|&b| b == 0xff);
|
||||
let sha256_digest = parts.nth(0).map(|b| b.to_vec()).unwrap();
|
||||
|
||||
self.filehash_metadata.remove(&sha256_digest)?;
|
||||
|
||||
let mut key = sha256_digest.clone();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(server_name.as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
self.filehash_servername_mediaid.remove(&key)?;
|
||||
|
||||
for (key, _) in self.filehash_thumbnailid.scan_prefix(sha256_digest.clone()) {
|
||||
self.filehash_thumbnailid.remove(&key)?;
|
||||
// 64 would be 0xff
|
||||
let (_, key) = key.split_at(65);
|
||||
self.thumbnailid_metadata.remove(key)?;
|
||||
}
|
||||
|
||||
Ok(hex::encode(sha256_digest))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn purge_and_get_hashes_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Result<String>> {
|
||||
let mut prefix = user_id.server_name().as_bytes().to_vec();
|
||||
prefix.push(0xff);
|
||||
prefix.extend_from_slice(user_id.localpart().as_bytes());
|
||||
prefix.push(0xff);
|
||||
|
||||
//TODO: rename this
|
||||
let stuff = self.purge_and_get_hashes(prefix, user_id.server_name(), after);
|
||||
|
||||
stuff
|
||||
.into_iter()
|
||||
.map(|r| match r {
|
||||
Ok((digest, media_id)) => {
|
||||
let mut key = user_id.server_name().as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.servernamemediaid_userlocalpart.remove(&key)?;
|
||||
|
||||
let mut key = user_id.server_name().as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(user_id.localpart().as_bytes());
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.servername_userlocalpart_mediaid.remove(&key)?;
|
||||
|
||||
Ok(hex::encode(digest))
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn purge_and_get_hashes_from_server(
|
||||
&self,
|
||||
server_name: &ServerName,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Result<String>> {
|
||||
let mut prefix = server_name.as_bytes().to_vec();
|
||||
prefix.push(0xff);
|
||||
|
||||
self.purge_and_get_hashes(prefix, server_name, after)
|
||||
.into_iter()
|
||||
.map(|r| r.map(|(digest, _)| hex::encode(digest)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn is_blocked(&self, server_name: &ServerName, media_id: &str) -> Result<bool> {
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.blocked_servername_mediaid
|
||||
.get(&key)
|
||||
.map(|x| x.is_some())
|
||||
}
|
||||
|
||||
fn block(
|
||||
&self,
|
||||
media: &[(OwnedServerName, String)],
|
||||
unix_secs: u64,
|
||||
reason: Option<String>,
|
||||
) -> Vec<Error> {
|
||||
let reason = reason.unwrap_or_default();
|
||||
let unix_secs = unix_secs.to_be_bytes();
|
||||
|
||||
media
|
||||
.iter()
|
||||
.map(|(server_name, media_id)| {
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
let mut value = unix_secs.to_vec();
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(reason.as_bytes());
|
||||
self.blocked_servername_mediaid.insert(&key, &value)
|
||||
})
|
||||
.filter_map(Result::err)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn block_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
now: u64,
|
||||
reason: &str,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Error> {
|
||||
let mut prefix = user_id.server_name().as_bytes().to_vec();
|
||||
prefix.push(0xff);
|
||||
prefix.extend_from_slice(user_id.localpart().as_bytes());
|
||||
prefix.push(0xff);
|
||||
|
||||
let mut value = now.to_be_bytes().to_vec();
|
||||
value.push(0xff);
|
||||
value.extend_from_slice(reason.as_bytes());
|
||||
|
||||
let media = self.servername_userlocalpart_mediaid.scan_prefix(prefix);
|
||||
|
||||
media
|
||||
.map(|(k, _)| {
|
||||
let parts = k.split(|&b| b == 0xff);
|
||||
|
||||
let media_id = parts.last().ok_or_else(|| {
|
||||
Error::bad_database("Invalid format of key in blocked_servername_mediaid")
|
||||
})?;
|
||||
|
||||
let mut key = user_id.server_name().as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id);
|
||||
|
||||
if after
|
||||
.map(|after| {
|
||||
let metadata =
|
||||
self.servernamemediaid_metadata.get(&key)?.ok_or_else(|| {
|
||||
Error::bad_database(
|
||||
"Missing metadata for local media in servernamemediaid_metadata",
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut parts = metadata.split(|&b| b == 0xff);
|
||||
|
||||
let sha256_digest = parts.next().ok_or_else(|| {
|
||||
Error::bad_database(
|
||||
"Invalid format of metadata in servernamemediaid_metadata",
|
||||
)
|
||||
})?;
|
||||
|
||||
let metadata =
|
||||
self.filehash_metadata.get(sha256_digest)?.ok_or_else(|| {
|
||||
Error::bad_database(
|
||||
"Missing metadata for local media in filehash_metadata",
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut parts = metadata.split(|&b| b == 0xff);
|
||||
|
||||
let creation_time = u64::from_be_bytes(
|
||||
parts
|
||||
.nth(1)
|
||||
.ok_or_else(|| {
|
||||
Error::bad_database(
|
||||
"Invalid format of metadata in filehash_metadata",
|
||||
)
|
||||
})?
|
||||
.try_into()
|
||||
.map_err(|_| {
|
||||
Error::bad_database(
|
||||
"Invalid creation time in filehash_metadata",
|
||||
)
|
||||
})?,
|
||||
);
|
||||
|
||||
Ok::<bool, Error>(creation_time > after)
|
||||
})
|
||||
.transpose()?
|
||||
.unwrap_or(true)
|
||||
{
|
||||
self.blocked_servername_mediaid.insert(&key, &value)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.filter_map(Result::err)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn unblock(&self, media: &[(OwnedServerName, String)]) -> Vec<Error> {
|
||||
media
|
||||
.iter()
|
||||
.map(|(server_name, media_id)| {
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.blocked_servername_mediaid.remove(&key)
|
||||
})
|
||||
.filter_map(Result::err)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn get_blocked(&self) -> Vec<Result<BlockedMediaInfo>> {
|
||||
self.blocked_servername_mediaid
|
||||
.iter()
|
||||
.map(|(k, v)| {
|
||||
let mut parts = k.split(|&b| b == 0xff);
|
||||
|
||||
let server_name = OwnedServerName::try_from(
|
||||
utils::string_from_bytes(parts.next().ok_or_else(|| {
|
||||
Error::bad_database("Invalid format of key in blocked_servername_mediaid")
|
||||
})?)
|
||||
.map_err(|_| {
|
||||
Error::bad_database(
|
||||
"Invalid server_name String in blocked_servername_mediaid",
|
||||
)
|
||||
})?,
|
||||
)
|
||||
.map_err(|_| {
|
||||
Error::bad_database("Invalid ServerName in blocked_servername_mediaid")
|
||||
})?;
|
||||
|
||||
let media_id = utils::string_from_bytes(parts.next().ok_or_else(|| {
|
||||
Error::bad_database("Invalid format of key in blocked_servername_mediaid")
|
||||
})?)
|
||||
.map_err(|_| {
|
||||
Error::bad_database(
|
||||
"Invalid media_id string in servername_userlocalpart_mediaid",
|
||||
)
|
||||
})?;
|
||||
|
||||
let mut parts = v.split(|&b| b == 0xff);
|
||||
|
||||
let unix_secs = u64::from_be_bytes(
|
||||
parts
|
||||
.next()
|
||||
.map(|bytes| {
|
||||
bytes.try_into().map_err(|_| {
|
||||
Error::bad_database(
|
||||
"Invalid block time in servername_userlocalpart_mediaid ",
|
||||
)
|
||||
})
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
Error::bad_database(
|
||||
"Invalid format of value in blocked_servername_mediaid",
|
||||
)
|
||||
})??,
|
||||
);
|
||||
|
||||
let reason = utils::string_from_bytes(parts.next().ok_or_else(|| {
|
||||
Error::bad_database("Invalid format of value in blocked_servername_mediaid")
|
||||
})?)
|
||||
.map_err(|_| {
|
||||
Error::bad_database(
|
||||
"Invalid media_id string in servername_userlocalpart_mediaid",
|
||||
)
|
||||
})?;
|
||||
|
||||
let reason = (!reason.is_empty()).then_some(reason);
|
||||
|
||||
Ok((server_name, media_id, unix_secs, reason))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl KeyValueDatabase {
|
||||
/// Purges all references to the given media in the database (apart from user localparts),
|
||||
/// returning a Vec of sha256 digests and media_ids
|
||||
fn purge_and_get_hashes(
|
||||
&self,
|
||||
prefix: Vec<u8>,
|
||||
server_name: &ServerName,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Result<(Vec<u8>, String)>> {
|
||||
let keys = self
|
||||
.servername_userlocalpart_mediaid
|
||||
.scan_prefix(prefix)
|
||||
.map(|(k, _)| k);
|
||||
|
||||
keys.map(|k| {
|
||||
let mut parts = k.split(|&b| b == 0xff);
|
||||
let x = parts.nth(2).map_or_else(
|
||||
|| {
|
||||
Err(Error::bad_database(
|
||||
"Invalid format for key of servername_userlocalpart_mediaid",
|
||||
))
|
||||
},
|
||||
|media_id_bytes| Ok(media_id_bytes.to_vec()),
|
||||
)?;
|
||||
|
||||
let media_id = utils::string_from_bytes(&x).map_err(|_| {
|
||||
Error::bad_database("Invalid media_id string in servername_userlocalpart_mediaid")
|
||||
})?;
|
||||
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
let metadata = self.servernamemediaid_metadata.get(&key)?.ok_or_else(|| {
|
||||
Error::bad_database("Missing metadata for media id and server_name")
|
||||
})?;
|
||||
let mut parts = metadata.split(|&b| b == 0xff);
|
||||
|
||||
let sha256_digest = parts.nth(0).map(|bytes| bytes.to_vec()).ok_or_else(|| {
|
||||
Error::bad_database("Invalid format of metadata in servernamemediaid_metadata")
|
||||
})?;
|
||||
|
||||
if if let Some(after) = after {
|
||||
let metadata = self.filehash_metadata.get(&sha256_digest)?.ok_or_else(|| {
|
||||
Error::bad_database("Missing metadata for media content sha256")
|
||||
})?;
|
||||
|
||||
let mut parts = metadata.split(|&b| b == 0xff);
|
||||
|
||||
let unix_secs = parts
|
||||
.nth(1)
|
||||
.map(|bytes| {
|
||||
bytes.try_into().map_err(|_| {
|
||||
Error::bad_database("Invalid creation time in filehash_metadata ")
|
||||
})
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
Error::bad_database("Invalid format of metadata in filehash_metadata")
|
||||
})??;
|
||||
|
||||
u64::from_be_bytes(unix_secs) > after
|
||||
} else {
|
||||
true
|
||||
} {
|
||||
self.filehash_metadata.remove(&sha256_digest)?;
|
||||
|
||||
for (key, _) in self
|
||||
.filehash_servername_mediaid
|
||||
.scan_prefix(sha256_digest.clone())
|
||||
{
|
||||
self.filehash_servername_mediaid.remove(&key)?;
|
||||
}
|
||||
|
||||
for (key, _) in self.filehash_thumbnailid.scan_prefix(sha256_digest.clone()) {
|
||||
self.filehash_thumbnailid.remove(&key)?;
|
||||
// 64 would be 0xff
|
||||
let (_, key) = key.split_at(65);
|
||||
self.thumbnailid_metadata.remove(key)?;
|
||||
}
|
||||
|
||||
let mut key = server_name.as_bytes().to_vec();
|
||||
key.push(0xff);
|
||||
key.extend_from_slice(media_id.as_bytes());
|
||||
|
||||
self.servernamemediaid_metadata.remove(&key)?;
|
||||
|
||||
Ok(Some((sha256_digest, media_id)))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
})
|
||||
.filter_map(Result::transpose)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,6 +149,20 @@ pub struct KeyValueDatabase {
|
|||
|
||||
//pub media: media::Media,
|
||||
pub(super) mediaid_file: Arc<dyn KvTree>, // MediaId = MXC + WidthHeight + ContentDisposition + ContentType
|
||||
// TODO: Passive provisions for MSC3911 (so that we don't need to do another migration or make a mess of the database), needing to consider:
|
||||
// - Restrictions over federation
|
||||
// - Deleting media when assocaited event is redacted
|
||||
// - Actually linking the media to an event
|
||||
//
|
||||
// https://github.com/matrix-org/matrix-spec-proposals/pull/3911
|
||||
pub(super) servernamemediaid_metadata: Arc<dyn KvTree>, // Servername + MediaID -> content sha256 + Filename + ContentType + extra 0xff byte if media is allowed on unauthenticated endpoints
|
||||
pub(super) filehash_servername_mediaid: Arc<dyn KvTree>, // sha256 of content + Servername + MediaID, used to delete dangling references to filehashes from servernamemediaid
|
||||
pub(super) filehash_metadata: Arc<dyn KvTree>, // sha256 of content -> file size + creation time + last access time
|
||||
pub(super) blocked_servername_mediaid: Arc<dyn KvTree>, // Servername + MediaID of blocked media
|
||||
pub(super) servername_userlocalpart_mediaid: Arc<dyn KvTree>, // Servername + User Localpart + MediaID
|
||||
pub(super) servernamemediaid_userlocalpart: Arc<dyn KvTree>, // Servername + MediaID -> User Localpart, used to remove keys from above when files are deleted by unrelated means
|
||||
pub(super) thumbnailid_metadata: Arc<dyn KvTree>, // ThumbnailId = Servername + MediaID + width + height -> Filename + ContentType + extra 0xff byte if media is allowed on unauthenticated endpoints
|
||||
pub(super) filehash_thumbnailid: Arc<dyn KvTree>, // sha256 of content + "ThumbnailId", as defined above. Used to dangling references to filehashes from thumbnailIds
|
||||
//pub key_backups: key_backups::KeyBackups,
|
||||
pub(super) backupid_algorithm: Arc<dyn KvTree>, // BackupId = UserId + Version(Count)
|
||||
pub(super) backupid_etag: Arc<dyn KvTree>, // BackupId = UserId + Version(Count)
|
||||
|
|
@ -352,7 +366,18 @@ impl KeyValueDatabase {
|
|||
referencedevents: builder.open_tree("referencedevents")?,
|
||||
roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,
|
||||
roomusertype_roomuserdataid: builder.open_tree("roomusertype_roomuserdataid")?,
|
||||
//TODO: Remove
|
||||
mediaid_file: builder.open_tree("mediaid_file")?,
|
||||
servernamemediaid_metadata: builder.open_tree("servernamemediaid_metadata")?,
|
||||
filehash_servername_mediaid: builder.open_tree("filehash_servername_mediaid")?,
|
||||
filehash_metadata: builder.open_tree("filehash_metadata")?,
|
||||
blocked_servername_mediaid: builder.open_tree("blocked_servername_mediaid")?,
|
||||
servername_userlocalpart_mediaid: builder
|
||||
.open_tree("servername_userlocalpart_mediaid")?,
|
||||
servernamemediaid_userlocalpart: builder
|
||||
.open_tree("servernamemediaid_userlocalpart")?,
|
||||
thumbnailid_metadata: builder.open_tree("thumbnailid_metadata")?,
|
||||
filehash_thumbnailid: builder.open_tree("filehash_thumbnailid")?,
|
||||
backupid_algorithm: builder.open_tree("backupid_algorithm")?,
|
||||
backupid_etag: builder.open_tree("backupid_etag")?,
|
||||
backupkeyid_backup: builder.open_tree("backupkeyid_backup")?,
|
||||
|
|
@ -415,7 +440,7 @@ impl KeyValueDatabase {
|
|||
}
|
||||
|
||||
// If the database has any data, perform data migrations before starting
|
||||
let latest_database_version = 16;
|
||||
let latest_database_version = 17;
|
||||
|
||||
if services().users.count()? > 0 {
|
||||
// MIGRATIONS
|
||||
|
|
@ -468,7 +493,9 @@ impl KeyValueDatabase {
|
|||
continue;
|
||||
}
|
||||
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let path = services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&key);
|
||||
let mut file = fs::File::create(path)?;
|
||||
file.write_all(&content)?;
|
||||
db.mediaid_file.insert(&key, &[])?;
|
||||
|
|
@ -936,7 +963,13 @@ impl KeyValueDatabase {
|
|||
// Reconstruct all media using the filesystem
|
||||
db.mediaid_file.clear().unwrap();
|
||||
|
||||
for file in fs::read_dir(services().globals.get_media_folder()).unwrap() {
|
||||
for file in fs::read_dir(
|
||||
services()
|
||||
.globals
|
||||
.get_media_folder_only_use_for_migrations(),
|
||||
)
|
||||
.unwrap()
|
||||
{
|
||||
let file = file.unwrap();
|
||||
let file_name = file.file_name().into_string().unwrap();
|
||||
|
||||
|
|
@ -1153,16 +1186,24 @@ fn migrate_content_disposition_format(
|
|||
|
||||
// Some file names are too long. Ignore those.
|
||||
match fs::rename(
|
||||
services().globals.get_media_file(&mediaid),
|
||||
services().globals.get_media_file(&new_key),
|
||||
services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&mediaid),
|
||||
services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&new_key),
|
||||
) {
|
||||
Ok(_) => {
|
||||
db.mediaid_file.insert(&new_key, &[])?;
|
||||
}
|
||||
Err(_) => {
|
||||
fs::rename(
|
||||
services().globals.get_media_file(&mediaid),
|
||||
services().globals.get_media_file(&shorter_key),
|
||||
services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&mediaid),
|
||||
services()
|
||||
.globals
|
||||
.get_media_file_old_only_use_for_migrations(&shorter_key),
|
||||
)
|
||||
.unwrap();
|
||||
db.mediaid_file.insert(&shorter_key, &[])?;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
use std::{collections::BTreeMap, convert::TryFrom, sync::Arc, time::Instant};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::BTreeMap,
|
||||
convert::TryFrom,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use clap::Parser;
|
||||
use chrono::DateTime;
|
||||
use clap::{Args, Parser};
|
||||
use regex::Regex;
|
||||
use ruma::{
|
||||
api::appservice::Registration,
|
||||
|
|
@ -19,8 +26,8 @@ use ruma::{
|
|||
},
|
||||
TimelineEventType,
|
||||
},
|
||||
EventId, MilliSecondsSinceUnixEpoch, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId,
|
||||
RoomVersionId, ServerName, UserId,
|
||||
EventId, MilliSecondsSinceUnixEpoch, MxcUri, OwnedRoomAliasId, OwnedRoomId, OwnedServerName,
|
||||
RoomAliasId, RoomId, RoomVersionId, ServerName, UserId,
|
||||
};
|
||||
use serde_json::value::to_raw_value;
|
||||
use tokio::sync::{mpsc, Mutex, RwLock};
|
||||
|
|
@ -82,11 +89,15 @@ enum AdminCommand {
|
|||
/// Deactivate a user
|
||||
///
|
||||
/// User will not be removed from all rooms by default.
|
||||
/// Use --leave-rooms to force the user to leave all rooms
|
||||
/// Use --leave-rooms to force the user to leave all rooms.
|
||||
/// Use either --purge-all-media or --purge-media-from-last to either delete all media uploaded
|
||||
/// by them (in the last <specified timeframe>, if any)
|
||||
DeactivateUser {
|
||||
#[arg(short, long)]
|
||||
leave_rooms: bool,
|
||||
user_id: Box<UserId>,
|
||||
#[command(flatten)]
|
||||
purge_media: PurgeMediaArgs,
|
||||
},
|
||||
|
||||
#[command(verbatim_doc_comment)]
|
||||
|
|
@ -94,6 +105,8 @@ enum AdminCommand {
|
|||
///
|
||||
/// Recommended to use in conjunction with list-local-users.
|
||||
///
|
||||
/// Use either --purge-all-media or --purge-media-from-last to either delete all media uploaded
|
||||
/// by them (in the last <specified timeframe>, if any)
|
||||
/// Users will not be removed from joined rooms by default.
|
||||
/// Can be overridden with --leave-rooms flag.
|
||||
/// Removing a mass amount of users from a room may cause a significant amount of leave events.
|
||||
|
|
@ -110,8 +123,67 @@ enum AdminCommand {
|
|||
#[arg(short, long)]
|
||||
/// Also deactivate admin accounts
|
||||
force: bool,
|
||||
#[command(flatten)]
|
||||
purge_media: PurgeMediaArgs,
|
||||
},
|
||||
|
||||
/// Purge a list of media, formatted as MXC URIs
|
||||
/// There should be one URI per line, all contained within a code-block
|
||||
PurgeMedia,
|
||||
|
||||
/// Purges all media uploaded by the local users listed in a code-block.
|
||||
/// Optionally, only delete media uploaded in the last <time-frame> with --from-last
|
||||
PurgeMediaFromUsers {
|
||||
#[arg(
|
||||
long, short,
|
||||
value_parser = humantime::parse_duration
|
||||
)]
|
||||
from_last: Option<Duration>,
|
||||
},
|
||||
|
||||
/// Purges all media from the specified server
|
||||
/// Optionally, only delete media downloaded in the last <time-frame> with --from-last
|
||||
PurgeMediaFromServer {
|
||||
server_id: Box<ServerName>,
|
||||
#[arg(
|
||||
long, short,
|
||||
value_parser = humantime::parse_duration
|
||||
)]
|
||||
from_last: Option<Duration>,
|
||||
},
|
||||
|
||||
/// Prevents the list of media from being accessed, but does not delete the media if it
|
||||
/// is already downloaded.
|
||||
/// There should be one MXC URI per line, all contained within a code-block
|
||||
BlockMedia {
|
||||
#[arg(long, short)]
|
||||
/// Prevents the specified media from being downloaded in the future
|
||||
and_purge: bool,
|
||||
#[arg(long, short)]
|
||||
/// Optional reason as to why this media should be blocked
|
||||
reason: Option<String>,
|
||||
},
|
||||
|
||||
/// Prevents all media uploaded by the local users, listed in a code-block, from being accessed
|
||||
/// Optionally, only block media uploaded in the last <time-frame> with --from-last
|
||||
BlockMediaFromUsers {
|
||||
#[arg(
|
||||
long, short,
|
||||
value_parser = humantime::parse_duration
|
||||
)]
|
||||
from_last: Option<Duration>,
|
||||
#[arg(long, short)]
|
||||
/// Optional reason as to why this media should be blocked
|
||||
reason: Option<String>,
|
||||
},
|
||||
|
||||
/// Lists all media that is currently blocked
|
||||
ListBlockedMedia,
|
||||
|
||||
/// Allows previously blocked media to be accessed again.
|
||||
/// There should be one MXC URI per line, all contained within a code-block
|
||||
UnblockMedia,
|
||||
|
||||
/// Get the auth_chain of a PDU
|
||||
GetAuthChain {
|
||||
/// An event ID (the $ character followed by the base64 reference hash)
|
||||
|
|
@ -181,6 +253,19 @@ enum AdminCommand {
|
|||
HashAndSignEvent { room_version_id: RoomVersionId },
|
||||
}
|
||||
|
||||
#[derive(Args, Debug)]
|
||||
#[group(required = false, multiple = false)]
|
||||
// Not an enum because https://github.com/clap-rs/clap/issues/2621
|
||||
pub struct PurgeMediaArgs {
|
||||
#[arg(long, short)]
|
||||
purge_all_media: bool,
|
||||
#[arg(
|
||||
long, short,
|
||||
value_parser = humantime::parse_duration
|
||||
)]
|
||||
purge_media_from_last: Option<Duration>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum AdminRoomEvent {
|
||||
ProcessMessage(String),
|
||||
|
|
@ -690,6 +775,7 @@ impl Service {
|
|||
AdminCommand::DeactivateUser {
|
||||
leave_rooms,
|
||||
user_id,
|
||||
purge_media,
|
||||
} => {
|
||||
let user_id = Arc::<UserId>::from(user_id);
|
||||
if !services().users.exists(&user_id)? {
|
||||
|
|
@ -711,78 +797,41 @@ impl Service {
|
|||
leave_all_rooms(&user_id).await?;
|
||||
}
|
||||
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"User {user_id} has been deactivated"
|
||||
let failed_purged_media = if purge_media.purge_media_from_last.is_some()
|
||||
|| purge_media.purge_all_media
|
||||
{
|
||||
let after = purge_media
|
||||
.purge_media_from_last
|
||||
.map(unix_secs_from_duration)
|
||||
.transpose()?;
|
||||
|
||||
services().media.purge_from_user(&user_id, after).len()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
if failed_purged_media == 0 {
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"User {user_id} has been deactivated"
|
||||
))
|
||||
} else {
|
||||
RoomMessageEventContent ::text_plain(format!(
|
||||
"User {user_id} has been deactivated, but {failed_purged_media} media failed to be purged, check the logs for more details"
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
AdminCommand::DeactivateAll { leave_rooms, force } => {
|
||||
AdminCommand::DeactivateAll {
|
||||
leave_rooms,
|
||||
force,
|
||||
purge_media,
|
||||
} => {
|
||||
if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```"
|
||||
{
|
||||
let users = body.clone().drain(1..body.len() - 1).collect::<Vec<_>>();
|
||||
|
||||
let mut user_ids = Vec::new();
|
||||
let mut remote_ids = Vec::new();
|
||||
let mut non_existent_ids = Vec::new();
|
||||
let mut invalid_users = Vec::new();
|
||||
|
||||
for &user in &users {
|
||||
match <&UserId>::try_from(user) {
|
||||
Ok(user_id) => {
|
||||
if user_id.server_name() != services().globals.server_name() {
|
||||
remote_ids.push(user_id)
|
||||
} else if !services().users.exists(user_id)? {
|
||||
non_existent_ids.push(user_id)
|
||||
} else {
|
||||
user_ids.push(user_id)
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
invalid_users.push(user);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut markdown_message = String::new();
|
||||
let mut html_message = String::new();
|
||||
if !invalid_users.is_empty() {
|
||||
markdown_message.push_str("The following user ids are not valid:\n```\n");
|
||||
html_message.push_str("The following user ids are not valid:\n<pre>\n");
|
||||
for invalid_user in invalid_users {
|
||||
markdown_message.push_str(&format!("{invalid_user}\n"));
|
||||
html_message.push_str(&format!("{invalid_user}\n"));
|
||||
}
|
||||
markdown_message.push_str("```\n\n");
|
||||
html_message.push_str("</pre>\n\n");
|
||||
}
|
||||
if !remote_ids.is_empty() {
|
||||
markdown_message
|
||||
.push_str("The following users are not from this server:\n```\n");
|
||||
html_message
|
||||
.push_str("The following users are not from this server:\n<pre>\n");
|
||||
for remote_id in remote_ids {
|
||||
markdown_message.push_str(&format!("{remote_id}\n"));
|
||||
html_message.push_str(&format!("{remote_id}\n"));
|
||||
}
|
||||
markdown_message.push_str("```\n\n");
|
||||
html_message.push_str("</pre>\n\n");
|
||||
}
|
||||
if !non_existent_ids.is_empty() {
|
||||
markdown_message.push_str("The following users do not exist:\n```\n");
|
||||
html_message.push_str("The following users do not exist:\n<pre>\n");
|
||||
for non_existent_id in non_existent_ids {
|
||||
markdown_message.push_str(&format!("{non_existent_id}\n"));
|
||||
html_message.push_str(&format!("{non_existent_id}\n"));
|
||||
}
|
||||
markdown_message.push_str("```\n\n");
|
||||
html_message.push_str("</pre>\n\n");
|
||||
}
|
||||
if !markdown_message.is_empty() {
|
||||
return Ok(RoomMessageEventContent::text_html(
|
||||
markdown_message,
|
||||
html_message,
|
||||
));
|
||||
}
|
||||
let mut user_ids = match userids_from_body(&body)? {
|
||||
Ok(v) => v,
|
||||
Err(message) => return Ok(message),
|
||||
};
|
||||
|
||||
let mut deactivation_count = 0;
|
||||
let mut admins = Vec::new();
|
||||
|
|
@ -812,12 +861,71 @@ impl Service {
|
|||
}
|
||||
}
|
||||
|
||||
if admins.is_empty() {
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"Deactivated {deactivation_count} accounts."
|
||||
let mut failed_count = 0;
|
||||
|
||||
if purge_media.purge_media_from_last.is_some() || purge_media.purge_all_media {
|
||||
let after = purge_media
|
||||
.purge_media_from_last
|
||||
.map(unix_secs_from_duration)
|
||||
.transpose()?;
|
||||
|
||||
for user_id in user_ids {
|
||||
failed_count += services().media.purge_from_user(user_id, after).len();
|
||||
}
|
||||
}
|
||||
|
||||
let mut message = format!("Deactivated {deactivation_count} accounts.");
|
||||
if !admins.is_empty() {
|
||||
message.push_str(&format!("\nSkipped admin accounts: {:?}. Use --force to deactivate admin accounts",admins.join(", ")));
|
||||
}
|
||||
if failed_count != 0 {
|
||||
message.push_str(&format!(
|
||||
"\nFailed to delete {failed_count} media, check logs for more details"
|
||||
))
|
||||
}
|
||||
|
||||
RoomMessageEventContent::text_plain(message)
|
||||
} else {
|
||||
RoomMessageEventContent::text_plain(
|
||||
"Expected code block in command body. Add --help for details.",
|
||||
)
|
||||
}
|
||||
}
|
||||
AdminCommand::PurgeMedia => media_from_body(body).map_or_else(
|
||||
|message| message,
|
||||
|media| {
|
||||
let failed_count = services().media.purge(&media).len();
|
||||
|
||||
if failed_count == 0 {
|
||||
RoomMessageEventContent::text_plain("Successfully purged media")
|
||||
} else {
|
||||
RoomMessageEventContent::text_plain(format!("Deactivated {} accounts.\nSkipped admin accounts: {:?}. Use --force to deactivate admin accounts", deactivation_count, admins.join(", ")))
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"Failed to delete {failed_count} media, check logs for more details"
|
||||
))
|
||||
}
|
||||
}),
|
||||
AdminCommand::PurgeMediaFromUsers { from_last } => {
|
||||
let after = from_last.map(unix_secs_from_duration).transpose()?;
|
||||
|
||||
if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```"
|
||||
{
|
||||
let user_ids = match userids_from_body(&body)? {
|
||||
Ok(v) => v,
|
||||
Err(message) => return Ok(message),
|
||||
};
|
||||
|
||||
let mut failed_count = 0;
|
||||
|
||||
for user_id in user_ids {
|
||||
failed_count += services().media.purge_from_user(user_id, after).len();
|
||||
}
|
||||
|
||||
if failed_count == 0 {
|
||||
RoomMessageEventContent::text_plain("Successfully purged media")
|
||||
} else {
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"Failed to purge {failed_count} media, check logs for more details"
|
||||
))
|
||||
}
|
||||
} else {
|
||||
RoomMessageEventContent::text_plain(
|
||||
|
|
@ -825,6 +933,141 @@ impl Service {
|
|||
)
|
||||
}
|
||||
}
|
||||
AdminCommand::PurgeMediaFromServer {
|
||||
server_id: server_name,
|
||||
from_last,
|
||||
} => {
|
||||
if server_name == services().globals.server_name() {
|
||||
return Err(Error::AdminCommand(
|
||||
"Cannot purge all media from your own homeserver",
|
||||
));
|
||||
}
|
||||
|
||||
let after = from_last.map(unix_secs_from_duration).transpose()?;
|
||||
|
||||
let failed_count = services()
|
||||
.media
|
||||
.purge_from_server(&server_name, after)
|
||||
.len();
|
||||
|
||||
if failed_count == 0 {
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"All media from {server_name} has successfully been purged"
|
||||
))
|
||||
} else {
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"Failed to purge {failed_count} media, check logs for more details"
|
||||
))
|
||||
}
|
||||
}
|
||||
AdminCommand::BlockMedia { and_purge, reason } => media_from_body(body).map_or_else(
|
||||
|message| message,
|
||||
|media| {
|
||||
let failed_count = services().media.block(&media, reason).len();
|
||||
let failed_purge_count = if and_purge {
|
||||
services().media.purge(&media).len()
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
match (failed_count == 0, failed_purge_count == 0) {
|
||||
(true, true) => RoomMessageEventContent::text_plain("Successfully blocked media"),
|
||||
(false, true) => RoomMessageEventContent::text_plain(format!(
|
||||
"Failed to block {failed_count} media, check logs for more details"
|
||||
)),
|
||||
(true, false ) => RoomMessageEventContent::text_plain(format!(
|
||||
"Failed to purge {failed_purge_count} media, check logs for more details"
|
||||
)),
|
||||
(false, false) => RoomMessageEventContent::text_plain(format!(
|
||||
"Failed to block {failed_count}, and purge {failed_purge_count} media, check logs for more details"
|
||||
))
|
||||
}
|
||||
},
|
||||
),
|
||||
AdminCommand::BlockMediaFromUsers { from_last, reason } => {
|
||||
let after = from_last.map(unix_secs_from_duration).transpose()?;
|
||||
|
||||
if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```"
|
||||
{
|
||||
let user_ids = match userids_from_body(&body)? {
|
||||
Ok(v) => v,
|
||||
Err(message) => return Ok(message),
|
||||
};
|
||||
|
||||
let mut failed_count = 0;
|
||||
|
||||
for user_id in user_ids {
|
||||
let reason = reason.as_ref().map_or_else(
|
||||
|| Cow::Owned(format!("uploaded by {user_id}")),
|
||||
Cow::Borrowed,
|
||||
);
|
||||
|
||||
failed_count += services()
|
||||
.media
|
||||
.block_from_user(user_id, &reason, after)
|
||||
.len();
|
||||
}
|
||||
|
||||
if failed_count == 0 {
|
||||
RoomMessageEventContent::text_plain("Successfully blocked media")
|
||||
} else {
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"Failed to block {failed_count} media, check logs for more details"
|
||||
))
|
||||
}
|
||||
} else {
|
||||
RoomMessageEventContent::text_plain(
|
||||
"Expected code block in command body. Add --help for details.",
|
||||
)
|
||||
}
|
||||
}
|
||||
AdminCommand::ListBlockedMedia => {
|
||||
let mut markdown_message = String::from(
|
||||
"| Server | Media ID | Time Blocked | Reason |\n| --- | --- | --- | --- |",
|
||||
);
|
||||
let mut html_message = String::from(
|
||||
r#"<table><thead><tr><th scope="col">Server</th><th scope="col">Media ID</th><th scope="col">Time Blocked</th><th scope="col">Reason</th></tr></thead><tbody>"#,
|
||||
);
|
||||
|
||||
for media in services().media.get_blocked() {
|
||||
let Ok((server, media_id, unix_secs, reason)) = media else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let reason = reason.unwrap_or_default();
|
||||
|
||||
let time = i64::try_from(unix_secs)
|
||||
.map(|unix_secs| DateTime::from_timestamp(unix_secs, 0))
|
||||
.ok()
|
||||
.flatten()
|
||||
.expect("Time is valid");
|
||||
|
||||
markdown_message
|
||||
.push_str(&format!("\n| {server} | {media_id} | {time} | {reason} |"));
|
||||
|
||||
html_message.push_str(&format!(
|
||||
"<tr><td>{server}</td><td>{media_id}</td><td>{time}</td><td>{reason}</td></tr>",
|
||||
))
|
||||
}
|
||||
|
||||
html_message.push_str("</tbody></table>");
|
||||
|
||||
RoomMessageEventContent::text_html(markdown_message, html_message)
|
||||
}
|
||||
AdminCommand::UnblockMedia => media_from_body(body).map_or_else(
|
||||
|message| message,
|
||||
|media| {
|
||||
let failed_count = services().media.unblock(&media).len();
|
||||
|
||||
if failed_count == 0 {
|
||||
RoomMessageEventContent::text_plain("Successfully unblocked media")
|
||||
} else {
|
||||
RoomMessageEventContent::text_plain(format!(
|
||||
"Failed to unblock {failed_count} media, check logs for more details"
|
||||
))
|
||||
}
|
||||
},
|
||||
),
|
||||
AdminCommand::SignJson => {
|
||||
if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```"
|
||||
{
|
||||
|
|
@ -1456,6 +1699,106 @@ impl Service {
|
|||
}
|
||||
}
|
||||
|
||||
fn userids_from_body<'a>(
|
||||
body: &'a [&'a str],
|
||||
) -> Result<Result<Vec<&'a UserId>, RoomMessageEventContent>, Error> {
|
||||
let users = body.to_owned().drain(1..body.len() - 1).collect::<Vec<_>>();
|
||||
|
||||
let mut user_ids = Vec::new();
|
||||
let mut remote_ids = Vec::new();
|
||||
let mut non_existent_ids = Vec::new();
|
||||
let mut invalid_users = Vec::new();
|
||||
|
||||
for &user in &users {
|
||||
match <&UserId>::try_from(user) {
|
||||
Ok(user_id) => {
|
||||
if user_id.server_name() != services().globals.server_name() {
|
||||
remote_ids.push(user_id)
|
||||
} else if !services().users.exists(user_id)? {
|
||||
non_existent_ids.push(user_id)
|
||||
} else {
|
||||
user_ids.push(user_id)
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
invalid_users.push(user);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut markdown_message = String::new();
|
||||
let mut html_message = String::new();
|
||||
if !invalid_users.is_empty() {
|
||||
markdown_message.push_str("The following user ids are not valid:\n```\n");
|
||||
html_message.push_str("The following user ids are not valid:\n<pre>\n");
|
||||
for invalid_user in invalid_users {
|
||||
markdown_message.push_str(&format!("{invalid_user}\n"));
|
||||
html_message.push_str(&format!("{invalid_user}\n"));
|
||||
}
|
||||
markdown_message.push_str("```\n\n");
|
||||
html_message.push_str("</pre>\n\n");
|
||||
}
|
||||
if !remote_ids.is_empty() {
|
||||
markdown_message.push_str("The following users are not from this server:\n```\n");
|
||||
html_message.push_str("The following users are not from this server:\n<pre>\n");
|
||||
for remote_id in remote_ids {
|
||||
markdown_message.push_str(&format!("{remote_id}\n"));
|
||||
html_message.push_str(&format!("{remote_id}\n"));
|
||||
}
|
||||
markdown_message.push_str("```\n\n");
|
||||
html_message.push_str("</pre>\n\n");
|
||||
}
|
||||
if !non_existent_ids.is_empty() {
|
||||
markdown_message.push_str("The following users do not exist:\n```\n");
|
||||
html_message.push_str("The following users do not exist:\n<pre>\n");
|
||||
for non_existent_id in non_existent_ids {
|
||||
markdown_message.push_str(&format!("{non_existent_id}\n"));
|
||||
html_message.push_str(&format!("{non_existent_id}\n"));
|
||||
}
|
||||
markdown_message.push_str("```\n\n");
|
||||
html_message.push_str("</pre>\n\n");
|
||||
}
|
||||
if !markdown_message.is_empty() {
|
||||
//TODO: This should be Err(Err(_))
|
||||
return Ok(Err(RoomMessageEventContent::text_html(
|
||||
markdown_message,
|
||||
html_message,
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(Ok(user_ids))
|
||||
}
|
||||
|
||||
fn media_from_body(
|
||||
body: Vec<&str>,
|
||||
) -> Result<Vec<(OwnedServerName, String)>, RoomMessageEventContent> {
|
||||
if body.len() > 2 && body[0].trim() == "```" && body.last().unwrap().trim() == "```" {
|
||||
Ok(body
|
||||
.clone()
|
||||
.drain(1..body.len() - 1)
|
||||
.map(<Box<MxcUri>>::from)
|
||||
.filter_map(|mxc| {
|
||||
mxc.parts()
|
||||
.map(|(server_name, media_id)| (server_name.to_owned(), media_id.to_owned()))
|
||||
.ok()
|
||||
})
|
||||
.collect::<Vec<_>>())
|
||||
} else {
|
||||
Err(RoomMessageEventContent::text_plain(
|
||||
"Expected code block in command body. Add --help for details.",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_secs_from_duration(duration: Duration) -> Result<u64> {
|
||||
SystemTime::now()
|
||||
.checked_sub(duration).ok_or_else(||Error::AdminCommand("Given timeframe cannot be represented as system time, please try again with a shorter time-frame"))
|
||||
.map(|time| time
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("Time is after unix epoch")
|
||||
.as_secs())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use ruma::{
|
|||
|
||||
use crate::api::server_server::DestinationResponse;
|
||||
|
||||
use crate::config::{DirectoryStructure, MediaConfig};
|
||||
use crate::{config::TurnConfig, services, Config, Error, Result};
|
||||
use futures_util::FutureExt;
|
||||
use hickory_resolver::TokioAsyncResolver;
|
||||
|
|
@ -227,7 +228,11 @@ impl Service {
|
|||
shutdown: AtomicBool::new(false),
|
||||
};
|
||||
|
||||
fs::create_dir_all(s.get_media_folder())?;
|
||||
// Remove this exception once other media backends are added
|
||||
#[allow(irrefutable_let_patterns)]
|
||||
if let MediaConfig::FileSystem { path, .. } = &s.config.media {
|
||||
fs::create_dir_all(path)?;
|
||||
}
|
||||
|
||||
if !s
|
||||
.supported_room_versions()
|
||||
|
|
@ -349,7 +354,18 @@ impl Service {
|
|||
}
|
||||
|
||||
pub fn turn(&self) -> Option<TurnConfig> {
|
||||
self.config.turn()
|
||||
// We have to clone basically the entire thing on `/turnServers` otherwise
|
||||
self.config.turn.clone()
|
||||
}
|
||||
|
||||
pub fn well_known_server(&self) -> OwnedServerName {
|
||||
// Same as above, but for /.well-known/matrix/server
|
||||
self.config.well_known.server.clone()
|
||||
}
|
||||
|
||||
pub fn well_known_client(&self) -> String {
|
||||
// Same as above, but for /.well-known/matrix/client
|
||||
self.config.well_known.client.clone()
|
||||
}
|
||||
|
||||
pub fn dns_resolver(&self) -> &TokioAsyncResolver {
|
||||
|
|
@ -466,14 +482,8 @@ impl Service {
|
|||
self.db.bump_database_version(new_version)
|
||||
}
|
||||
|
||||
pub fn get_media_folder(&self) -> PathBuf {
|
||||
let mut r = PathBuf::new();
|
||||
r.push(self.config.database_path.clone());
|
||||
r.push("media");
|
||||
r
|
||||
}
|
||||
|
||||
pub fn get_media_file(&self, key: &[u8]) -> PathBuf {
|
||||
/// As the name states, old version of `get_media_file`, only for usage in migrations
|
||||
pub fn get_media_file_old_only_use_for_migrations(&self, key: &[u8]) -> PathBuf {
|
||||
let mut r = PathBuf::new();
|
||||
r.push(self.config.database_path.clone());
|
||||
r.push("media");
|
||||
|
|
@ -481,12 +491,41 @@ impl Service {
|
|||
r
|
||||
}
|
||||
|
||||
pub fn well_known_server(&self) -> OwnedServerName {
|
||||
self.config.well_known_server()
|
||||
/// As the name states, this should only be used for migrations.
|
||||
pub fn get_media_folder_only_use_for_migrations(&self) -> PathBuf {
|
||||
let mut r = PathBuf::new();
|
||||
r.push(self.config.database_path.clone());
|
||||
r.push("media");
|
||||
r
|
||||
}
|
||||
|
||||
pub fn well_known_client(&self) -> String {
|
||||
self.config.well_known_client()
|
||||
//TODO: Separate directory for remote media?
|
||||
pub fn get_media_path(
|
||||
&self,
|
||||
media_directory: &str,
|
||||
directory_structure: &DirectoryStructure,
|
||||
sha256_hex: &str,
|
||||
) -> Result<PathBuf> {
|
||||
let mut r = PathBuf::new();
|
||||
r.push(media_directory);
|
||||
|
||||
if let DirectoryStructure::Deep { length, depth } = directory_structure {
|
||||
let mut filename = sha256_hex;
|
||||
for _ in 0..depth.get() {
|
||||
let (current_path, next) = filename.split_at(length.get().into());
|
||||
filename = next;
|
||||
r.push(current_path);
|
||||
}
|
||||
|
||||
// Create all directories leading up to file
|
||||
fs::create_dir_all(&r)?;
|
||||
|
||||
r.push(filename);
|
||||
} else {
|
||||
r.push(sha256_hex);
|
||||
}
|
||||
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
pub fn shutdown(&self) {
|
||||
|
|
|
|||
|
|
@ -1,22 +1,91 @@
|
|||
use ruma::http_headers::ContentDisposition;
|
||||
use ruma::{OwnedServerName, ServerName, UserId};
|
||||
use sha2::{digest::Output, Sha256};
|
||||
|
||||
use crate::Result;
|
||||
use crate::{Error, Result};
|
||||
|
||||
use super::BlockedMediaInfo;
|
||||
|
||||
pub trait Data: Send + Sync {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn create_file_metadata(
|
||||
&self,
|
||||
mxc: String,
|
||||
width: u32,
|
||||
height: u32,
|
||||
content_disposition: &ContentDisposition,
|
||||
sha256_digest: Output<Sha256>,
|
||||
file_size: u64,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<Vec<u8>>;
|
||||
user_id: Option<&UserId>,
|
||||
) -> Result<()>;
|
||||
|
||||
/// Returns content_disposition, content_type and the metadata key.
|
||||
/// Returns the sha256 hash, filename, content_type and whether the media should be accessible via
|
||||
/// unauthenticated endpoints.
|
||||
fn search_file_metadata(
|
||||
&self,
|
||||
mxc: String,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
) -> Result<(String, Option<String>, Option<String>, bool)>;
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn create_thumbnail_metadata(
|
||||
&self,
|
||||
sha256_digest: Output<Sha256>,
|
||||
file_size: u64,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<(ContentDisposition, Option<String>, Vec<u8>)>;
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<()>;
|
||||
|
||||
// Returns the sha256 hash, filename and content_type and whether the media should be accessible via
|
||||
/// unauthenticated endpoints.
|
||||
fn search_thumbnail_metadata(
|
||||
&self,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<(String, Option<String>, Option<String>, bool)>;
|
||||
|
||||
fn purge_and_get_hashes(&self, media: &[(OwnedServerName, String)]) -> Vec<Result<String>>;
|
||||
|
||||
fn purge_and_get_hashes_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Result<String>>;
|
||||
|
||||
fn purge_and_get_hashes_from_server(
|
||||
&self,
|
||||
server_name: &ServerName,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Result<String>>;
|
||||
|
||||
fn is_blocked(&self, server_name: &ServerName, media_id: &str) -> Result<bool>;
|
||||
|
||||
fn block(
|
||||
&self,
|
||||
media: &[(OwnedServerName, String)],
|
||||
unix_secs: u64,
|
||||
reason: Option<String>,
|
||||
) -> Vec<Error>;
|
||||
|
||||
fn block_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
now: u64,
|
||||
reason: &str,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Error>;
|
||||
|
||||
fn unblock(&self, media: &[(OwnedServerName, String)]) -> Vec<Error>;
|
||||
|
||||
/// Returns a Vec of:
|
||||
/// - The server the media is from
|
||||
/// - The media id
|
||||
/// - The time it was blocked, in unix seconds
|
||||
/// - The optional reason why it was blocked
|
||||
fn get_blocked(&self) -> Vec<Result<BlockedMediaInfo>>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,20 @@
|
|||
mod data;
|
||||
use std::io::Cursor;
|
||||
use std::{fs, io::Cursor};
|
||||
|
||||
pub use data::Data;
|
||||
use ruma::{
|
||||
api::client::error::ErrorKind,
|
||||
api::client::{error::ErrorKind, media::is_safe_inline_content_type},
|
||||
http_headers::{ContentDisposition, ContentDispositionType},
|
||||
OwnedServerName, ServerName, UserId,
|
||||
};
|
||||
use sha2::{digest::Output, Digest, Sha256};
|
||||
|
||||
use crate::{services, Result};
|
||||
use crate::{config::MediaConfig, services, utils, Error, Result};
|
||||
use image::imageops::FilterType;
|
||||
|
||||
use tokio::{
|
||||
fs::File,
|
||||
io::{AsyncReadExt, AsyncWriteExt, BufReader},
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
};
|
||||
|
||||
pub struct FileMeta {
|
||||
|
|
@ -25,73 +27,86 @@ pub struct Service {
|
|||
pub db: &'static dyn Data,
|
||||
}
|
||||
|
||||
pub type BlockedMediaInfo = (OwnedServerName, String, u64, Option<String>);
|
||||
|
||||
impl Service {
|
||||
/// Uploads a file.
|
||||
pub async fn create(
|
||||
&self,
|
||||
mxc: String,
|
||||
content_disposition: Option<ContentDisposition>,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
file: &[u8],
|
||||
user_id: Option<&UserId>,
|
||||
) -> Result<()> {
|
||||
let content_disposition =
|
||||
content_disposition.unwrap_or(ContentDisposition::new(ContentDispositionType::Inline));
|
||||
let (sha256_digest, sha256_hex) = generate_digests(file);
|
||||
|
||||
// Width, Height = 0 if it's not a thumbnail
|
||||
let key = self
|
||||
.db
|
||||
.create_file_metadata(mxc, 0, 0, &content_disposition, content_type)?;
|
||||
self.db.create_file_metadata(
|
||||
sha256_digest,
|
||||
size(file)?,
|
||||
servername,
|
||||
media_id,
|
||||
filename,
|
||||
content_type,
|
||||
user_id,
|
||||
)?;
|
||||
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut f = File::create(path).await?;
|
||||
f.write_all(file).await?;
|
||||
Ok(())
|
||||
create_file(&sha256_hex, file).await
|
||||
}
|
||||
|
||||
/// Uploads or replaces a file thumbnail.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn upload_thumbnail(
|
||||
&self,
|
||||
mxc: String,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
filename: Option<&str>,
|
||||
content_type: Option<&str>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
file: &[u8],
|
||||
) -> Result<()> {
|
||||
let key = self.db.create_file_metadata(
|
||||
mxc,
|
||||
let (sha256_digest, sha256_hex) = generate_digests(file);
|
||||
|
||||
self.db.create_thumbnail_metadata(
|
||||
sha256_digest,
|
||||
size(file)?,
|
||||
servername,
|
||||
media_id,
|
||||
width,
|
||||
height,
|
||||
&ContentDisposition::new(ContentDispositionType::Inline),
|
||||
filename,
|
||||
content_type,
|
||||
)?;
|
||||
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut f = File::create(path).await?;
|
||||
f.write_all(file).await?;
|
||||
|
||||
Ok(())
|
||||
create_file(&sha256_hex, file).await
|
||||
}
|
||||
|
||||
/// Downloads a file.
|
||||
pub async fn get(&self, mxc: String) -> Result<Option<FileMeta>> {
|
||||
if let Ok((content_disposition, content_type, key)) =
|
||||
self.db.search_file_metadata(mxc, 0, 0)
|
||||
{
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut file = Vec::new();
|
||||
BufReader::new(File::open(path).await?)
|
||||
.read_to_end(&mut file)
|
||||
.await?;
|
||||
/// Fetches a local file and it's metadata
|
||||
pub async fn get(
|
||||
&self,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
authenticated: bool,
|
||||
) -> Result<Option<FileMeta>> {
|
||||
let Ok((sha256, filename, content_type, unauthenticated_access_permitted)) =
|
||||
self.db.search_file_metadata(servername, media_id)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file,
|
||||
}))
|
||||
} else {
|
||||
Ok(None)
|
||||
if !(authenticated || unauthenticated_access_permitted) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let file = get_file(&sha256).await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition: content_disposition(filename, &content_type),
|
||||
content_type,
|
||||
file,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Returns width, height of the thumbnail and whether it should be cropped. Returns None when
|
||||
|
|
@ -119,117 +134,326 @@ impl Service {
|
|||
/// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
|
||||
pub async fn get_thumbnail(
|
||||
&self,
|
||||
mxc: String,
|
||||
servername: &ServerName,
|
||||
media_id: &str,
|
||||
width: u32,
|
||||
height: u32,
|
||||
authenticated: bool,
|
||||
) -> Result<Option<FileMeta>> {
|
||||
let (width, height, crop) = self
|
||||
.thumbnail_properties(width, height)
|
||||
.unwrap_or((0, 0, false)); // 0, 0 because that's the original file
|
||||
|
||||
if let Ok((content_disposition, content_type, key)) =
|
||||
self.db.search_file_metadata(mxc.clone(), width, height)
|
||||
{
|
||||
// Using saved thumbnail
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut file = Vec::new();
|
||||
File::open(path).await?.read_to_end(&mut file).await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file: file.to_vec(),
|
||||
}))
|
||||
} else if let Ok((content_disposition, content_type, key)) =
|
||||
self.db.search_file_metadata(mxc.clone(), 0, 0)
|
||||
{
|
||||
// Generate a thumbnail
|
||||
let path = services().globals.get_media_file(&key);
|
||||
let mut file = Vec::new();
|
||||
File::open(path).await?.read_to_end(&mut file).await?;
|
||||
|
||||
if let Ok(image) = image::load_from_memory(&file) {
|
||||
let original_width = image.width();
|
||||
let original_height = image.height();
|
||||
if width > original_width || height > original_height {
|
||||
return Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file: file.to_vec(),
|
||||
}));
|
||||
if let Some((width, height, crop)) = self.thumbnail_properties(width, height) {
|
||||
if let Ok((sha256, filename, content_type, unauthenticated_access_permitted)) = self
|
||||
.db
|
||||
.search_thumbnail_metadata(servername, media_id, width, height)
|
||||
{
|
||||
if !(authenticated || unauthenticated_access_permitted) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let thumbnail = if crop {
|
||||
image.resize_to_fill(width, height, FilterType::CatmullRom)
|
||||
} else {
|
||||
let (exact_width, exact_height) = {
|
||||
// Copied from image::dynimage::resize_dimensions
|
||||
let ratio = u64::from(original_width) * u64::from(height);
|
||||
let nratio = u64::from(width) * u64::from(original_height);
|
||||
|
||||
let use_width = nratio <= ratio;
|
||||
let intermediate = if use_width {
|
||||
u64::from(original_height) * u64::from(width)
|
||||
/ u64::from(original_width)
|
||||
} else {
|
||||
u64::from(original_width) * u64::from(height)
|
||||
/ u64::from(original_height)
|
||||
};
|
||||
if use_width {
|
||||
if intermediate <= u64::from(u32::MAX) {
|
||||
(width, intermediate as u32)
|
||||
} else {
|
||||
(
|
||||
(u64::from(width) * u64::from(u32::MAX) / intermediate) as u32,
|
||||
u32::MAX,
|
||||
)
|
||||
}
|
||||
} else if intermediate <= u64::from(u32::MAX) {
|
||||
(intermediate as u32, height)
|
||||
} else {
|
||||
(
|
||||
u32::MAX,
|
||||
(u64::from(height) * u64::from(u32::MAX) / intermediate) as u32,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
image.thumbnail_exact(exact_width, exact_height)
|
||||
};
|
||||
|
||||
let mut thumbnail_bytes = Vec::new();
|
||||
thumbnail.write_to(
|
||||
&mut Cursor::new(&mut thumbnail_bytes),
|
||||
image::ImageFormat::Png,
|
||||
)?;
|
||||
|
||||
// Save thumbnail in database so we don't have to generate it again next time
|
||||
let thumbnail_key = self.db.create_file_metadata(
|
||||
mxc,
|
||||
width,
|
||||
height,
|
||||
&content_disposition,
|
||||
content_type.as_deref(),
|
||||
)?;
|
||||
|
||||
let path = services().globals.get_media_file(&thumbnail_key);
|
||||
let mut f = File::create(path).await?;
|
||||
f.write_all(&thumbnail_bytes).await?;
|
||||
// Using saved thumbnail
|
||||
let file = get_file(&sha256).await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_disposition: content_disposition(filename, &content_type),
|
||||
content_type,
|
||||
file: thumbnail_bytes.to_vec(),
|
||||
file,
|
||||
}))
|
||||
} else {
|
||||
// Couldn't parse file to generate thumbnail, likely not an image
|
||||
return Err(crate::Error::BadRequest(
|
||||
} else if let Ok((sha256, filename, content_type, unauthenticated_access_permitted)) =
|
||||
self.db.search_file_metadata(servername, media_id)
|
||||
{
|
||||
if !(authenticated || unauthenticated_access_permitted) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let content_disposition = content_disposition(filename.clone(), &content_type);
|
||||
// Generate a thumbnail
|
||||
let file = get_file(&sha256).await?;
|
||||
|
||||
if let Ok(image) = image::load_from_memory(&file) {
|
||||
let original_width = image.width();
|
||||
let original_height = image.height();
|
||||
if width > original_width || height > original_height {
|
||||
return Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file,
|
||||
}));
|
||||
}
|
||||
|
||||
let thumbnail = if crop {
|
||||
image.resize_to_fill(width, height, FilterType::CatmullRom)
|
||||
} else {
|
||||
let (exact_width, exact_height) = {
|
||||
// Copied from image::dynimage::resize_dimensions
|
||||
let ratio = u64::from(original_width) * u64::from(height);
|
||||
let nratio = u64::from(width) * u64::from(original_height);
|
||||
|
||||
let use_width = nratio <= ratio;
|
||||
let intermediate = if use_width {
|
||||
u64::from(original_height) * u64::from(width)
|
||||
/ u64::from(original_width)
|
||||
} else {
|
||||
u64::from(original_width) * u64::from(height)
|
||||
/ u64::from(original_height)
|
||||
};
|
||||
if use_width {
|
||||
if intermediate <= u64::from(u32::MAX) {
|
||||
(width, intermediate as u32)
|
||||
} else {
|
||||
(
|
||||
(u64::from(width) * u64::from(u32::MAX) / intermediate)
|
||||
as u32,
|
||||
u32::MAX,
|
||||
)
|
||||
}
|
||||
} else if intermediate <= u64::from(u32::MAX) {
|
||||
(intermediate as u32, height)
|
||||
} else {
|
||||
(
|
||||
u32::MAX,
|
||||
(u64::from(height) * u64::from(u32::MAX) / intermediate) as u32,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
image.thumbnail_exact(exact_width, exact_height)
|
||||
};
|
||||
|
||||
let mut thumbnail_bytes = Vec::new();
|
||||
thumbnail.write_to(
|
||||
&mut Cursor::new(&mut thumbnail_bytes),
|
||||
image::ImageFormat::Png,
|
||||
)?;
|
||||
|
||||
// Save thumbnail in database so we don't have to generate it again next time
|
||||
self.upload_thumbnail(
|
||||
servername,
|
||||
media_id,
|
||||
filename.as_deref(),
|
||||
content_type.as_deref(),
|
||||
width,
|
||||
height,
|
||||
&thumbnail_bytes,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition,
|
||||
content_type,
|
||||
file: thumbnail_bytes,
|
||||
}))
|
||||
} else {
|
||||
// Couldn't parse file to generate thumbnail, likely not an image
|
||||
return Err(Error::BadRequest(
|
||||
ErrorKind::Unknown,
|
||||
"Unable to generate thumbnail for the requested content (likely is not an image)",
|
||||
));
|
||||
}
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
} else {
|
||||
Ok(None)
|
||||
// Using full-sized file
|
||||
let Ok((sha256, filename, content_type, unauthenticated_access_permitted)) =
|
||||
self.db.search_file_metadata(servername, media_id)
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if !(authenticated || unauthenticated_access_permitted) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let file = get_file(&sha256).await?;
|
||||
|
||||
Ok(Some(FileMeta {
|
||||
content_disposition: content_disposition(filename, &content_type),
|
||||
content_type,
|
||||
file,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// Purges all of the specified media.
|
||||
///
|
||||
/// Returns errors for all the files that were failed to be deleted, if any.
|
||||
pub fn purge(&self, media: &[(OwnedServerName, String)]) -> Vec<Error> {
|
||||
let hashes = self.db.purge_and_get_hashes(media);
|
||||
|
||||
purge_files(hashes)
|
||||
}
|
||||
|
||||
/// Purges all (past a certain time in unix seconds, if specified) media
|
||||
/// sent by a user.
|
||||
///
|
||||
/// Returns errors for all the files that were failed to be deleted, if any.
|
||||
///
|
||||
/// Note: it only currently works for local users, as we cannot determine who
|
||||
/// exactly uploaded the file when it comes to remove users.
|
||||
pub fn purge_from_user(&self, user_id: &UserId, after: Option<u64>) -> Vec<Error> {
|
||||
let hashes = self.db.purge_and_get_hashes_from_user(user_id, after);
|
||||
|
||||
purge_files(hashes)
|
||||
}
|
||||
|
||||
/// Purges all (past a certain time in unix seconds, if specified) media
|
||||
/// obtained from the specified server (due to the MXC URI).
|
||||
///
|
||||
/// Returns errors for all the files that were failed to be deleted, if any.
|
||||
pub fn purge_from_server(&self, server_name: &ServerName, after: Option<u64>) -> Vec<Error> {
|
||||
let hashes = self.db.purge_and_get_hashes_from_server(server_name, after);
|
||||
|
||||
purge_files(hashes)
|
||||
}
|
||||
|
||||
/// Checks whether the media has been blocked by administrators, returning either
|
||||
/// a database error, or a not found error if it is blocked
|
||||
pub fn check_blocked(&self, server_name: &ServerName, media_id: &str) -> Result<()> {
|
||||
if self.db.is_blocked(server_name, media_id)? {
|
||||
Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks the specified media as blocked, preventing them from being accessed
|
||||
pub fn block(&self, media: &[(OwnedServerName, String)], reason: Option<String>) -> Vec<Error> {
|
||||
let now = utils::secs_since_unix_epoch();
|
||||
|
||||
self.db.block(media, now, reason)
|
||||
}
|
||||
|
||||
/// Marks the media uploaded by a local user as blocked, preventing it from being accessed
|
||||
pub fn block_from_user(
|
||||
&self,
|
||||
user_id: &UserId,
|
||||
reason: &str,
|
||||
after: Option<u64>,
|
||||
) -> Vec<Error> {
|
||||
let now = utils::secs_since_unix_epoch();
|
||||
|
||||
self.db.block_from_user(user_id, now, reason, after)
|
||||
}
|
||||
|
||||
/// Unblocks the specified media, allowing them from being accessed again
|
||||
pub fn unblock(&self, media: &[(OwnedServerName, String)]) -> Vec<Error> {
|
||||
self.db.unblock(media)
|
||||
}
|
||||
|
||||
/// Returns a Vec of:
|
||||
/// - The server the media is from
|
||||
/// - The media id
|
||||
/// - The time it was blocked, in unix seconds
|
||||
/// - The optional reason why it was blocked
|
||||
pub fn get_blocked(&self) -> Vec<Result<BlockedMediaInfo>> {
|
||||
self.db.get_blocked()
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates the media file, using the configured media backend
|
||||
///
|
||||
/// Note: this function does NOT set the metadata related to the file
|
||||
async fn create_file(sha256_hex: &str, file: &[u8]) -> Result<()> {
|
||||
match &services().globals.config.media {
|
||||
MediaConfig::FileSystem {
|
||||
path,
|
||||
directory_structure,
|
||||
} => {
|
||||
let path = services()
|
||||
.globals
|
||||
.get_media_path(path, directory_structure, sha256_hex)?;
|
||||
|
||||
let mut f = File::create(path).await?;
|
||||
f.write_all(file).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fetches the file from the configured media backend
|
||||
async fn get_file(sha256_hex: &str) -> Result<Vec<u8>> {
|
||||
Ok(match &services().globals.config.media {
|
||||
MediaConfig::FileSystem {
|
||||
path,
|
||||
directory_structure,
|
||||
} => {
|
||||
let path = services()
|
||||
.globals
|
||||
.get_media_path(path, directory_structure, sha256_hex)?;
|
||||
|
||||
let mut file = Vec::new();
|
||||
File::open(path).await?.read_to_end(&mut file).await?;
|
||||
|
||||
file
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Purges the given files from the media backend
|
||||
/// Returns a `Vec` of errors that occurred when attempting to delete the files
|
||||
///
|
||||
/// Note: this does NOT remove the related metadata from the database
|
||||
fn purge_files(hashes: Vec<Result<String>>) -> Vec<Error> {
|
||||
hashes
|
||||
.into_iter()
|
||||
.map(|hash| match hash {
|
||||
Ok(v) => delete_file(&v),
|
||||
Err(e) => Err(e),
|
||||
})
|
||||
.filter_map(|r| if let Err(e) = r { Some(e) } else { None })
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Deletes the given file from the media backend
|
||||
///
|
||||
/// Note: this does NOT remove the related metadata from the database
|
||||
fn delete_file(sha256_hex: &str) -> Result<()> {
|
||||
match &services().globals.config.media {
|
||||
MediaConfig::FileSystem {
|
||||
path,
|
||||
directory_structure,
|
||||
} => {
|
||||
let path = services()
|
||||
.globals
|
||||
.get_media_path(path, directory_structure, sha256_hex)?;
|
||||
|
||||
fs::remove_file(path)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a content disposition with the given `filename`, using the `content_type` to determine whether
|
||||
/// the disposition should be `inline` or `attachment`
|
||||
fn content_disposition(
|
||||
filename: Option<String>,
|
||||
content_type: &Option<String>,
|
||||
) -> ContentDisposition {
|
||||
ContentDisposition::new(
|
||||
if content_type
|
||||
.as_deref()
|
||||
.is_some_and(is_safe_inline_content_type)
|
||||
{
|
||||
ContentDispositionType::Inline
|
||||
} else {
|
||||
ContentDispositionType::Attachment
|
||||
},
|
||||
)
|
||||
.with_filename(filename)
|
||||
}
|
||||
|
||||
/// Returns sha256 digests of the file, in raw (Vec) and hex form respectively
|
||||
fn generate_digests(file: &[u8]) -> (Output<Sha256>, String) {
|
||||
let sha256_digest = Sha256::digest(file);
|
||||
let hex_sha256 = hex::encode(sha256_digest);
|
||||
|
||||
(sha256_digest, hex_sha256)
|
||||
}
|
||||
|
||||
/// Get's the file size, is bytes, as u64, returning an error if the file size is larger
|
||||
/// than a u64 (which is far too big to be reasonably uploaded in the first place anyways)
|
||||
fn size(file: &[u8]) -> Result<u64> {
|
||||
u64::try_from(file.len())
|
||||
.map_err(|_| Error::BadRequest(ErrorKind::TooLarge, "File is too large"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,13 @@ pub fn millis_since_unix_epoch() -> u64 {
|
|||
.as_millis() as u64
|
||||
}
|
||||
|
||||
pub fn secs_since_unix_epoch() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("time is valid")
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
pub fn increment(old: Option<&[u8]>) -> Option<Vec<u8>> {
|
||||
let number = match old.map(|bytes| bytes.try_into()) {
|
||||
Some(Ok(bytes)) => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue