generate Sitemap

Cargo.toml

@@ -42,6 +42,7 @@ rhai = { version = "1.23.6", features = ["only_i32", "f32_float", "internals", "
 schemars = "1.0.4"
 serde = { version = "1.0.219", features = ["derive"] }
 serde_json = "1.0.145"
+sitemap-rs = "0.3.0"
 slug = "0.1.6"
 smartstring = "1.0.1"
 syntect = "5.2.0"

poet/Cargo.toml

@@ -45,6 +45,7 @@ rhai_components = { path = "../rhai_components", version = "0.5" }
 schemars = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+sitemap-rs = { workspace = true }
 slug = { workspace = true }
 smartstring = { workspace = true }
 syntect = { workspace = true }

poet/src/build_project/build_project_params.rs

@@ -10,6 +10,7 @@ pub struct BuildProjectParams {
     pub asset_path_renderer: AssetPathRenderer,
     pub esbuild_metafile: Arc<EsbuildMetaFile>,
     pub generated_page_base_path: String,
+    pub generate_sitemap: bool,
     pub is_watching: bool,
     pub rhai_template_renderer: RhaiTemplateRenderer,
     pub source_filesystem: Arc<Storage>,

poet/src/build_project/mod.rs

@@ -7,6 +7,7 @@ mod content_document_rendering_context;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
 use std::collections::HashSet;
+use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -40,6 +41,7 @@ use crate::filesystem::Filesystem as _;
 use crate::filesystem::memory::Memory;
 use crate::find_front_matter_in_mdast::find_front_matter_in_mdast;
 use crate::find_table_of_contents_in_mdast::find_table_of_contents_in_mdast;
+use crate::generate_sitemap::create_sitemap;
 use crate::string_to_mdast::string_to_mdast;
 
 fn render_document<'render>(
@@ -101,11 +103,12 @@ fn render_document<'render>(
 
 pub async fn build_project(
     BuildProjectParams {
-        asset_path_renderer,
+        mut asset_path_renderer,
         esbuild_metafile,
         generated_page_base_path,
         is_watching,
         rhai_template_renderer,
+        generate_sitemap,
         source_filesystem,
     }: BuildProjectParams,
 ) -> Result<BuildProjectResultStub> {
@@ -259,7 +262,7 @@ pub async fn build_project(
     let content_document_reference_collection_dashmap: DashMap<String, ContentDocumentReference> =
         Default::default();
     let content_document_basename_by_id_arc = Arc::new(content_document_basename_by_id);
-    let content_document_by_basename_arc = Arc::new(content_document_by_basename);
+    let content_document_by_basename_arc = Arc::new(content_document_by_basename.clone());
     let content_document_collections_ranked_arc = Arc::new(content_document_collections_ranked);
     let content_document_linker = ContentDocumentLinker {
         content_document_basename_by_id: content_document_basename_by_id_arc.clone(),
@@ -323,6 +326,26 @@ pub async fn build_project(
             }
         });
 
+    if generate_sitemap {
+        info!("Building sitemap");
+
+        match create_sitemap(
+            &mut asset_path_renderer.base_path,
+            content_document_by_basename.values(),
+        ) {
+            Ok(sitemap) => {
+                if let Err(err) =
+                    memory_filesystem.set_file_contents_sync(&Path::new("sitemap.xml"), &sitemap)
+                {
+                    error_collection.register_error("sitemap.xml".to_string(), err);
+                }
+            }
+            Err(err) => {
+                error_collection.register_error("sitemap.xml".to_string(), err);
+            }
+        }
+    }
+
     if error_collection.is_empty() {
         Ok(BuildProjectResultStub {
             esbuild_metafile,

poet/src/cmd/make/static_pages.rs

@@ -27,6 +27,9 @@ pub struct StaticPages {
     #[arg(long)]
     public_path: String,
 
+    #[arg(long, default_value = "false")]
+    sitemap: bool,
+
     #[arg(value_parser = validate_is_directory)]
     source_directory: PathBuf,
 }
@@ -55,6 +58,7 @@ impl Handler for StaticPages {
             generated_page_base_path: self.public_path.clone(),
             is_watching: false,
             rhai_template_renderer,
+            generate_sitemap: self.sitemap,
             source_filesystem,
         })
         .await?;

poet/src/cmd/serve/mod.rs

@@ -10,9 +10,9 @@ use actix_web::App;
 use actix_web::HttpServer;
 use actix_web::web::Data;
 use anyhow::Result;
-use indoc::formatdoc;
 use async_trait::async_trait;
 use clap::Parser;
+use indoc::formatdoc;
 use log::info;
 
 use crate::app_dir_desktop_entry::AppDirDesktopEntry;
@@ -60,6 +60,9 @@ pub struct Serve {
 
     #[arg(long)]
     public_path: String,
+
+    #[arg(long, default_value = "false")]
+    sitemap: bool,
 }
 
 impl BuildsProject for Serve {
@@ -114,6 +117,7 @@ impl Handler for Serve {
             generated_page_base_path: self.public_path.clone(),
             is_watching: false,
             rhai_template_renderer: rhai_template_renderer.clone(),
+            generate_sitemap: self.sitemap,
             source_filesystem: source_filesystem.clone(),
         })
         .await?

poet/src/cmd/watch/mod.rs

@@ -47,6 +47,9 @@ pub struct Watch {
 
     #[arg(value_parser = validate_is_directory)]
     source_directory: PathBuf,
+
+    #[arg(long, default_value = "false")]
+    sitemap: bool,
 }
 
 impl BuildsProject for Watch {
@@ -134,6 +137,7 @@ impl Handler for Watch {
             on_content_file_changed,
             rhai_template_renderer_holder: rhai_template_renderer_holder.clone(),
             session_manager,
+            generate_sitemap: self.sitemap,
             source_filesystem: source_filesystem.clone(),
         }));
 

poet/src/cmd/watch/service/filesystem_http_route_index_builder.rs

@@ -20,7 +20,7 @@ pub struct FilesystemHttpRouteIndexBuilder {
 }
 
 impl FilesystemHttpRouteIndexBuilder {
-    async fn do_build_filesystem_htto_route_index(&self) {
+    async fn do_build_filesystem_http_route_index(&self) {
         let BuildProjectResult {
             memory_filesystem, ..
         } = match self.build_project_result_holder.get().await {
@@ -51,7 +51,7 @@ impl FilesystemHttpRouteIndexBuilder {
 impl Service for FilesystemHttpRouteIndexBuilder {
     async fn run(&self) -> Result<()> {
         loop {
-            self.do_build_filesystem_htto_route_index().await;
+            self.do_build_filesystem_http_route_index().await;
 
             tokio::select! {
                 _ = self.build_project_result_holder.update_notifier.notified() => continue,

poet/src/cmd/watch/service/project_builder.rs

@@ -31,6 +31,7 @@ pub struct ProjectBuilder {
     pub on_content_file_changed: Arc<Notify>,
     pub rhai_template_renderer_holder: RhaiTemplateRendererHolder,
     pub session_manager: SessionManager,
+    pub generate_sitemap: bool,
     pub source_filesystem: Arc<Storage>,
 }
 
@@ -60,6 +61,7 @@ impl ProjectBuilder {
             generated_page_base_path: self.generated_page_base_path.clone(),
             is_watching: true,
             rhai_template_renderer,
+            generate_sitemap: self.generate_sitemap,
             source_filesystem: self.source_filesystem.clone(),
         })
         .await

poet/src/filesystem_http_route_index.rs

@@ -29,7 +29,7 @@ impl FilesystemHttpRouteIndex {
         Ok(this)
     }
 
-    pub fn register_file(&self, file: FileEntry) -> Result<()> {
+    fn register_file(&self, file: FileEntry) -> Result<()> {
         let filename = file.relative_path.to_string_lossy().to_string();
 
         if filename.ends_with("/index.html") {
@@ -43,6 +43,8 @@ impl FilesystemHttpRouteIndex {
         } else if filename == "index.html" {
             self.routes.insert("".to_string(), file.clone());
             self.routes.insert("index.html".to_string(), file.clone());
+        } else if filename == "sitemap.xml" {
+            self.routes.insert(filename, file.clone());
         } else {
             return Err(anyhow!("Unexpected filename: '{filename}'"));
         }

poet/src/generate_sitemap.rs

@@ -0,0 +1,59 @@
+use anyhow::{Error, Result};
+use chrono::Utc;
+use sitemap_rs::url::Url;
+use sitemap_rs::url_set::UrlSet;
+
+use crate::content_document_basename::ContentDocumentBasename;
+use crate::content_document_reference::ContentDocumentReference;
+
+pub fn create_sitemap(
+    base_url: &mut String,
+    content_document_by_basename: std::collections::hash_map::Values<
+        '_,
+        ContentDocumentBasename,
+        ContentDocumentReference,
+    >,
+) -> Result<String, Error> {
+    let last_modified = Utc::now().fixed_offset();
+    let mut urls: Vec<Url> = vec![Url::new(
+        base_url.clone(),
+        Some(last_modified),
+        None,
+        Some(0.8),
+        None,
+        None,
+        None,
+    )?];
+
+    for reference in content_document_by_basename {
+        let mut page_path = reference
+            .basename_path
+            .to_string_lossy()
+            .into_owned()
+            .replace("index", "");
+
+        if !base_url.ends_with('/') {
+            base_url.push_str("/");
+        }
+
+        if page_path != "" {
+            page_path = format!("{base_url}{page_path}");
+
+            urls.push(Url::new(
+                page_path,
+                Some(last_modified),
+                None,
+                Some(0.5),
+                None,
+                None,
+                None,
+            )?);
+        }
+    }
+
+    let url_set: UrlSet = UrlSet::new(urls)?;
+    let mut buf: Vec<u8> = Vec::<u8>::new();
+    url_set.write(&mut buf).unwrap();
+
+    Ok(String::from_utf8(buf)?)
+}

poet/src/lib.rs

@@ -37,6 +37,7 @@ pub mod find_front_matter_in_mdast;
 pub mod find_table_of_contents_in_mdast;
 pub mod find_text_content_in_mdast;
 pub mod flexible_datetime;
+pub mod generate_sitemap;
 pub mod holder;
 pub mod is_external_link;
 pub mod is_valid_desktop_entry_string;

poet/src/search_index.rs

@@ -119,6 +119,7 @@ mod tests {
             generated_page_base_path: public_path,
             is_watching: false,
             rhai_template_renderer,
+            generate_sitemap: false,
             source_filesystem,
         })
         .await

rhai_components/src/component_syntax/eval_tag_stack_node.rs

@@ -111,7 +111,11 @@ pub fn eval_tag_stack_node(
                 Ok(rhai_call_template_function(
                     eval_context.engine(),
                     &opening_tag.tag_name.name,
-                    (context, Dynamic::from_map(props), Dynamic::from(result.to_string())),
+                    (
+                        context,
+                        Dynamic::from_map(props),
+                        Dynamic::from(result.to_string()),
+                    ),
                 )
                 .map_err(|err| {
                     EvalAltResult::ErrorRuntime(

rhai_components/src/rhai_call_template_function.rs

@@ -19,7 +19,8 @@ pub fn rhai_call_template_function(
 
     let tmp_ast = AST::new([], module);
 
-    let result = engine.call_fn::<ImmutableString>(&mut Scope::new(), &tmp_ast, "template", args)?;
+    let result =
+        engine.call_fn::<ImmutableString>(&mut Scope::new(), &tmp_ast, "template", args)?;
 
     Ok(result.into())
 }