feat(media): deep hashed directory structure

docs/configuration.md

@@ -64,7 +64,7 @@ The `global` section contains the following fields:
 ### Media
 The `media` table is used to configure how media is stored and where. Currently, there is only one available
 backend, that being `filesystem`. The backend can be set using the `backend` field. Example:
-```
+```toml
 [global.media]
 backend = "filesystem" # the default backend
 ```
@@ -73,12 +73,30 @@ backend = "filesystem" # the default backend
 The filesystem backend has the following fields:
 - `path`: The base directory where all the media files will be stored (defaults to
   `${database_path}/media`)
+- `directory_structure`: This is a table, used to configure how files are to be distributed within
+  the media directory. It has the following fields:
+  - `depth`: The number sub-directories that should be created for files (default: `1`)
+  - `length`: How long the name of these sub-directories should be (default: `4`)
+  For example, a file may regularly have the name `98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4`
+  (The SHA256 digest of the file's content). If `depth` and `length` were both set to `2`, this file would be stored
+  at `${path}/98/ea/6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4`. If you want to instead have all
+  media files in the base directory with no sub-directories, just set `directory_structure` to be empty, as follows:
+  ```toml
+  [global.media]
+  backend = "filesystem"
+
+  [global.media.directory_structure]
+  ```
 
 ##### Example:
-```
+```toml
 [global.media]
 backend = "filesystem"
 path = "/srv/matrix-media"
+
+[global.media.directory_structure]
+depth = 4
+length = 2
 ```
 
 ### TLS

src/config/mod.rs

@@ -2,6 +2,7 @@ use std::{
     collections::BTreeMap,
     fmt,
     net::{IpAddr, Ipv4Addr},
+    num::NonZeroU8,
     path::PathBuf,
 };
 
@@ -10,10 +11,13 @@ use serde::{de::IgnoredAny, Deserialize};
 use tracing::warn;
 use url::Url;
 
-mod proxy;
+use crate::Error;
 
+mod proxy;
 use self::proxy::ProxyConfig;
 
+const SHA256_HEX_LENGTH: u8 = 64;
+
 #[derive(Deserialize)]
 pub struct IncompleteConfig {
     #[serde(default = "default_address")]
@@ -218,7 +222,10 @@ impl From<IncompleteConfig> for Config {
         };
 
         let media = match media {
-            IncompleteMediaConfig::FileSystem { path } => MediaConfig::FileSystem {
+            IncompleteMediaConfig::FileSystem {
+                path,
+                directory_structure,
+            } => MediaConfig::FileSystem {
                 path: path.unwrap_or_else(|| {
                     // We do this as we don't know if the path has a trailing slash, or even if the
                     // path separator is a forward or backward slash
@@ -229,6 +236,7 @@ impl From<IncompleteConfig> for Config {
                         .into_string()
                         .expect("Both inputs are valid UTF-8")
                 }),
+                directory_structure,
             },
         };
 
@@ -309,21 +317,85 @@ pub struct WellKnownConfig {
     pub server: OwnedServerName,
 }
 
-#[derive(Clone, Debug, Deserialize)]
+#[derive(Deserialize)]
 #[serde(tag = "backend", rename_all = "lowercase")]
 pub enum IncompleteMediaConfig {
-    FileSystem { path: Option<String> },
+    FileSystem {
+        path: Option<String>,
+        #[serde(default)]
+        directory_structure: DirectoryStructure,
+    },
 }
 
 impl Default for IncompleteMediaConfig {
     fn default() -> Self {
-        Self::FileSystem { path: None }
+        Self::FileSystem {
+            path: None,
+            directory_structure: DirectoryStructure::default(),
+        }
     }
 }
 
 #[derive(Debug, Clone)]
 pub enum MediaConfig {
-    FileSystem { path: String },
+    FileSystem {
+        path: String,
+        directory_structure: DirectoryStructure,
+    },
+}
+
+#[derive(Debug, Clone, Deserialize)]
+// See https://github.com/serde-rs/serde/issues/642#issuecomment-525432907
+#[serde(try_from = "ShadowDirectoryStructure", untagged)]
+pub enum DirectoryStructure {
+    // We do this enum instead of Option<DirectoryStructure>, so that we can have the structure be
+    // deep by default, while still providing a away for it to be flat (by creating an empty table)
+    //
+    // e.g.:
+    // ```toml
+    // [global.media.directory_structure]
+    // ```
+    Flat,
+    Deep { length: NonZeroU8, depth: NonZeroU8 },
+}
+
+impl Default for DirectoryStructure {
+    fn default() -> Self {
+        Self::Deep {
+            length: NonZeroU8::new(4).expect("4 is not 0"),
+            depth: NonZeroU8::new(1).expect("1 is not 0"),
+        }
+    }
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum ShadowDirectoryStructure {
+    Flat,
+    Deep { length: NonZeroU8, depth: NonZeroU8 },
+}
+
+impl TryFrom<ShadowDirectoryStructure> for DirectoryStructure {
+    type Error = Error;
+
+    fn try_from(value: ShadowDirectoryStructure) -> Result<Self, Self::Error> {
+        match value {
+            ShadowDirectoryStructure::Flat => Ok(Self::Flat),
+            ShadowDirectoryStructure::Deep { length, depth } => {
+                if length
+                    .get()
+                    .checked_mul(depth.get())
+                    .map(|product| product < SHA256_HEX_LENGTH)
+                    // If an overflow occurs, it definitely isn't less than SHA256_HEX_LENGTH
+                    .unwrap_or(false)
+                {
+                    Ok(Self::Deep { length, depth })
+                } else {
+                    Err(Error::bad_config("The media directory structure depth multiplied by the depth is equal to or greater than a sha256 hex hash, please reduce at least one of the two so that their product is less than 64"))
+                }
+            }
+        }
+    }
 }
 
 const DEPRECATED_KEYS: &[&str] = &[

src/service/globals/mod.rs

@@ -7,7 +7,7 @@ use ruma::{
 
 use crate::api::server_server::DestinationResponse;
 
-use crate::config::MediaConfig;
+use crate::config::{DirectoryStructure, MediaConfig};
 use crate::{config::TurnConfig, services, Config, Error, Result};
 use futures_util::FutureExt;
 use hickory_resolver::TokioAsyncResolver;
@@ -230,7 +230,7 @@ impl Service {
 
         // Remove this exception once other media backends are added
         #[allow(irrefutable_let_patterns)]
-        if let MediaConfig::FileSystem { path } = &s.config.media {
+        if let MediaConfig::FileSystem { path, .. } = &s.config.media {
             fs::create_dir_all(path)?;
         }
 
@@ -500,14 +500,32 @@ impl Service {
     }
 
     //TODO: Separate directory for remote media?
-    pub fn get_media_path(&self, media_directory: &str, sha256_hex: &str) -> PathBuf {
+    pub fn get_media_path(
+        &self,
+        media_directory: &str,
+        directory_structure: &DirectoryStructure,
+        sha256_hex: &str,
+    ) -> Result<PathBuf> {
         let mut r = PathBuf::new();
         r.push(media_directory);
 
-        //TODO: Directory distribution
-        r.push(sha256_hex);
+        if let DirectoryStructure::Deep { length, depth } = directory_structure {
+            let mut filename = sha256_hex;
+            for _ in 0..depth.get() {
+                let (current_path, next) = filename.split_at(length.get().into());
+                filename = next;
+                r.push(current_path);
+            }
 
-        r
+            // Create all directories leading up to file
+            fs::create_dir_all(&r)?;
+
+            r.push(filename);
+        } else {
+            r.push(sha256_hex);
+        }
+
+        Ok(r)
     }
 
     pub fn shutdown(&self) {

src/service/media/mod.rs

@@ -275,8 +275,13 @@ impl Service {
 /// Note: this function does NOT set the metadata related to the file
 async fn create_file(sha256_hex: &str, file: &[u8]) -> Result<()> {
     match &services().globals.config.media {
-        MediaConfig::FileSystem { path } => {
-            let path = services().globals.get_media_path(path, sha256_hex);
+        MediaConfig::FileSystem {
+            path,
+            directory_structure,
+        } => {
+            let path = services()
+                .globals
+                .get_media_path(path, directory_structure, sha256_hex)?;
 
             let mut f = File::create(path).await?;
             f.write_all(file).await?;
@@ -289,8 +294,13 @@ async fn create_file(sha256_hex: &str, file: &[u8]) -> Result<()> {
 /// Fetches the file from the configured media backend
 async fn get_file(sha256_hex: &str) -> Result<Vec<u8>> {
     Ok(match &services().globals.config.media {
-        MediaConfig::FileSystem { path } => {
-            let path = services().globals.get_media_path(path, sha256_hex);
+        MediaConfig::FileSystem {
+            path,
+            directory_structure,
+        } => {
+            let path = services()
+                .globals
+                .get_media_path(path, directory_structure, sha256_hex)?;
 
             let mut file = Vec::new();
             File::open(path).await?.read_to_end(&mut file).await?;