Add a first approximation of full text search. Issue #8

assets/style.css

@@ -307,3 +307,16 @@ h1>input {
         box-shadow: 2px 2px 8px rgba(0,0,0, 0.25);
     }
 }
+
+article ul.search-results {
+    padding-left: 8px;
+    list-style: none;
+}
+.search-result {
+    border: 1px solid #ccc;
+    padding: 8px 16px;
+    margin-bottom: 16px;
+}
+.snippet {
+    white-space: pre-line;
+}

build.rs

@@ -29,8 +29,17 @@ fn main() {
 
     let infer_schema_path = Path::new(&out_dir).join("infer_schema.rs");
     let mut file = File::create(infer_schema_path).expect("Unable to open file for writing");
+
     file.write_all(quote! {
-        infer_schema!(#db_path);
+        mod __diesel_infer_schema_articles {
+            infer_table_from_schema!(#db_path, "articles");
+        }
+        pub use self::__diesel_infer_schema_articles::*;
+
+        mod __diesel_infer_schema_article_revisions {
+            infer_table_from_schema!(#db_path, "article_revisions");
+        }
+        pub use self::__diesel_infer_schema_article_revisions::*;
     }.as_str().as_bytes()).expect("Unable to write to file");
 
     for entry in WalkDir::new("migrations").into_iter().filter_map(|e| e.ok()) {

migrations/20171024122359_add_article_fts_table/down.sql

@@ -0,0 +1,6 @@
+DROP TRIGGER article_revisions_ai;
+DROP TRIGGER article_revisions_ad;
+DROP TRIGGER article_revisions_au_disable;
+DROP TRIGGER article_revisions_au_enable;
+
+DROP TABLE article_search;

migrations/20171024122359_add_article_fts_table/up.sql

@@ -0,0 +1,25 @@
+CREATE VIRTUAL TABLE article_search USING fts5(
+    title,
+    body,
+    slug UNINDEXED,
+    tokenize = 'porter unicode61 remove_diacritics 1'
+);
+
+INSERT INTO article_search(title, body, slug)
+    SELECT title, body, slug FROM article_revisions WHERE latest != 0;
+
+CREATE TRIGGER article_revisions_ai AFTER INSERT ON article_revisions WHEN new.latest != 0 BEGIN
+  DELETE FROM article_search WHERE rowid = new.article_id;
+  INSERT INTO article_search(rowid, title, body, slug) VALUES (new.article_id, new.title, new.body, new.slug);
+END;
+CREATE TRIGGER article_revisions_ad AFTER DELETE ON article_revisions WHEN old.latest != 0 BEGIN
+  DELETE FROM article_search WHERE rowid = old.article_id;
+END;
+
+-- Index unique_latest_revision_per_article_id makes sure the following is sufficient:
+CREATE TRIGGER article_revisions_au_disable AFTER UPDATE ON article_revisions WHEN old.latest != 0 AND new.latest == 0 BEGIN
+  DELETE FROM article_search WHERE rowid = old.article_id;
+END;
+CREATE TRIGGER article_revisions_au_enable AFTER UPDATE ON article_revisions WHEN old.latest == 0 AND new.latest != 0 BEGIN
+  INSERT INTO article_search(rowid, title, body, slug) VALUES (new.article_id, new.title, new.body, new.slug);
+END;

src/models.rs

@@ -32,3 +32,10 @@ pub struct ArticleRevisionStub {
 
     pub author: Option<String>,
 }
+
+#[derive(Debug, Queryable)]
+pub struct SearchResult {
+    pub title: String,
+    pub snippet: String,
+    pub slug: String,
+}

src/resources/mod.rs

@@ -5,6 +5,7 @@ mod article_revision_resource;
 mod article_resource;
 mod changes_resource;
 mod new_article_resource;
+mod search_resource;
 mod sitemap_resource;
 mod temporary_redirect_resource;
 
@@ -13,5 +14,6 @@ pub use self::article_revision_resource::ArticleRevisionResource;
 pub use self::article_resource::ArticleResource;
 pub use self::changes_resource::{ChangesLookup, ChangesResource};
 pub use self::new_article_resource::NewArticleResource;
+pub use self::search_resource::SearchLookup;
 pub use self::sitemap_resource::SitemapResource;
 pub use self::temporary_redirect_resource::TemporaryRedirectResource;

src/resources/search_resource.rs

@@ -0,0 +1,129 @@
+use futures::{self, Future};
+use hyper;
+use hyper::header::ContentType;
+use hyper::server::*;
+use serde_urlencoded;
+
+use assets::StyleCss;
+use mimes::*;
+use models;
+use site::Layout;
+use state::State;
+use web::{Resource, ResponseFuture};
+
+const DEFAULT_LIMIT: i32 = 30;
+
+type BoxResource = Box<Resource + Sync + Send>;
+
+#[derive(Serialize, Deserialize, Default)]
+pub struct QueryParameters {
+    q: Option<String>,
+    skip: Option<i32>,
+    limit: Option<i32>,
+}
+
+impl QueryParameters {
+    pub fn skip(self, skip: i32) -> Self {
+        Self {
+            skip: if skip != 0 { Some(skip) } else { None },
+            ..self
+        }
+    }
+
+    pub fn limit(self, limit: i32) -> Self {
+        Self {
+            limit: if limit != DEFAULT_LIMIT { Some(limit) } else { None },
+            ..self
+        }
+    }
+
+    pub fn into_link(self) -> String {
+        let args = serde_urlencoded::to_string(self).expect("Serializing to String cannot fail");
+        if args.len() > 0 {
+            format!("_search?{}", args)
+        } else {
+            "_search".to_owned()
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct SearchLookup {
+    state: State,
+}
+
+impl SearchLookup {
+    pub fn new(state: State) -> Self {
+        Self { state }
+    }
+
+    pub fn lookup(&self, query: Option<&str>) -> Result<Option<BoxResource>, ::web::Error> {
+        let args: QueryParameters = serde_urlencoded::from_str(query.unwrap_or(""))?;
+
+        Ok(Some(Box::new(SearchResource::new(self.state.clone(), args))))
+    }
+}
+
+pub struct SearchResource {
+    state: State,
+    query_args: QueryParameters,
+}
+
+impl SearchResource {
+    pub fn new(state: State, query_args: QueryParameters) -> Self {
+        Self { state, query_args }
+    }
+}
+
+impl Resource for SearchResource {
+    fn allow(&self) -> Vec<hyper::Method> {
+        use hyper::Method::*;
+        vec![Options, Head, Get]
+    }
+
+    fn head(&self) -> ResponseFuture {
+        Box::new(futures::finished(Response::new()
+            .with_status(hyper::StatusCode::Ok)
+            .with_header(ContentType(TEXT_HTML.clone()))
+        ))
+    }
+
+    fn get(self: Box<Self>) -> ResponseFuture {
+        #[derive(BartDisplay)]
+        #[template="templates/search.html"]
+        struct Template<'a> {
+            query: &'a str,
+            hits: Vec<models::SearchResult>,
+        }
+
+        impl models::SearchResult {
+            fn link(&self) -> String {
+                if self.slug == "" {
+                    ".".to_owned()
+                } else {
+                    self.slug.clone()
+                }
+            }
+        }
+
+        // TODO: Show a search "front page" when no query is given:
+        let query = self.query_args.q.as_ref().map(|x| x.clone()).unwrap_or("".to_owned());
+
+        let data = self.state.search_query(query);
+        let head = self.head();
+
+        Box::new(data.join(head)
+            .and_then(move |(data, head)| {
+                Ok(head
+                    .with_body(Layout {
+                        base: None, // Hmm, should perhaps accept `base` as argument
+                        title: "Search",
+                        body: &Template {
+                            query: self.query_args.q.as_ref().map(|x| &**x).unwrap_or(""),
+                            hits: data,
+                        },
+                        style_css_checksum: StyleCss::checksum(),
+                    }.to_string()))
+            }))
+    }
+}

src/state.rs

@@ -319,4 +319,24 @@ impl State {
             })
         })
     }
+
+    pub fn search_query(&self, query_string: String) -> CpuFuture<Vec<models::SearchResult>, Error> {
+        let connection_pool = self.connection_pool.clone();
+
+        self.cpu_pool.spawn_fn(move || {
+            use diesel::expression::sql_literal::sql;
+            use diesel::types::Text;
+
+            Ok(
+                sql::<(Text, Text, Text)>(
+                    "SELECT title, snippet(article_search, 1, '', '', '\u{2026}', 8), slug \
+                        FROM article_search \
+                        WHERE article_search MATCH ?
+                        ORDER BY rank"
+                )
+                .bind::<Text, _>(query_string)
+                .load(&*connection_pool.get()?)?)
+        })
+    }
+
 }

src/wiki_lookup.rs

@@ -2,7 +2,7 @@ use std::borrow::Cow;
 use std::collections::HashMap;
 use std::str::Utf8Error;
 
-use futures::{Future, finished, failed};
+use futures::{Future, finished, failed, done};
 use futures::future::FutureResult;
 use percent_encoding::percent_decode;
 use slug::slugify;
@@ -42,6 +42,7 @@ lazy_static! {
 pub struct WikiLookup {
     state: State,
     changes_lookup: ChangesLookup,
+    search_lookup: SearchLookup,
 }
 
 fn split_one(path: &str) -> Result<(Cow<str>, Option<&str>), Utf8Error> {
@@ -72,8 +73,9 @@ fn asset_lookup(path: &str) -> FutureResult<Option<BoxResource>, Box<::std::erro
 impl WikiLookup {
     pub fn new(state: State) -> WikiLookup {
         let changes_lookup = ChangesLookup::new(state.clone());
+        let search_lookup = SearchLookup::new(state.clone());
 
-        WikiLookup { state, changes_lookup }
+        WikiLookup { state, changes_lookup, search_lookup }
     }
 
     fn revisions_lookup(&self, path: &str, _query: Option<&str>) -> <Self as Lookup>::Future {
@@ -140,6 +142,8 @@ impl WikiLookup {
                 Box::new(finished(Some(Box::new(NewArticleResource::new(self.state.clone(), None)) as BoxResource))),
             ("_revisions", Some(tail)) =>
                 self.revisions_lookup(tail, query),
+            ("_search", None) =>
+                Box::new(done(self.search_lookup.lookup(query))),
             ("_sitemap", None) =>
                 Box::new(finished(Some(Box::new(SitemapResource::new(self.state.clone())) as BoxResource))),
             _ => Box::new(finished(None)),

templates/search.html

@@ -0,0 +1,26 @@
+<div class="container">
+<header>
+<h1>Search</h1>
+</header>
+
+<article>
+<form method="GET"><input autofocus name="q" autocomplete=off value="{{query}}"></form>
+
+{{#hits?}}
+<p>Search results for the query <b>{{query}}</b>:</p>
+
+<ul class="search-results">
+    {{#hits}}
+    <li class="search-result"><a href="{{.link()}}">{{.title}}</a> &ndash; <span class="snippet">{{.snippet}}</span></li>
+    {{/hits}}
+</ul>
+{{/hits}}
+
+{{^hits?}}
+<p>Your search for <b>{{query}}</b> gave no results.</p>
+{{/hits}}
+
+</article>
+</div>
+
+{{>footer/default.html}}