From d199d679c46d44c8e55940450cbf53ec7e8c2767 Mon Sep 17 00:00:00 2001 From: djkato Date: Mon, 15 Jul 2024 16:04:35 +0200 Subject: [PATCH] attempt to fix UrlUpdates creating new urls instead of updating --- sitemap-generator/Cargo.toml | 2 +- .../src/sitemap/event_handler.rs | 8 +- sitemap-generator/src/sitemap/mod.rs | 19 +- sitemap-generator/src/tests/mod.rs | 15 + sitemap-generator/src/tests/utils.rs | 311 ++++++------------ 5 files changed, 142 insertions(+), 213 deletions(-) diff --git a/sitemap-generator/Cargo.toml b/sitemap-generator/Cargo.toml index 3664f91..7fb46af 100644 --- a/sitemap-generator/Cargo.toml +++ b/sitemap-generator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "sitemap-generator" -version = "1.0.0" +version = "1.0.1" edition = "2021" authors = ["Djkáťo "] description = "Creates and keeps Sitemap.xml uptodate with Saleor." diff --git a/sitemap-generator/src/sitemap/event_handler.rs b/sitemap-generator/src/sitemap/event_handler.rs index 093d2b4..be69ddf 100644 --- a/sitemap-generator/src/sitemap/event_handler.rs +++ b/sitemap-generator/src/sitemap/event_handler.rs @@ -16,14 +16,14 @@ use crate::{ sitemap::Url, }; use tokio::{sync::mpsc::Receiver, task::JoinHandle}; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, info, warn}; use super::{ItemData, ItemType, UrlSet}; // 10k links google says, but there's also a size limit and my custom params might be messing with // that? Rather split prematurely to be sure. const MAX_URL_IN_SET: usize = 50_000; -const DB_FILE_NAME: &str = "db.toml"; +const DB_FILE_NAME: &str = "db.json"; const SITEMAP_FILE_NAME: &str = "sitemap.txt"; pub struct EventHandler { @@ -297,7 +297,7 @@ async fn update_or_create( debug!("affected urls: {:?}", &affected_urls); if affected_urls.is_empty() { - trace!("{:?} doesn't exist in url_set yet", &item.slug); + debug!("{:?} doesn't exist in url_set yet", &item.slug); url_set.push(Url::new(data, sitemap_config, item, rel_item).unwrap()); } else { // Update affected urls @@ -353,7 +353,7 @@ async fn delete(id: &str, sitemap_config: &SitemapConfig) { /* =================== File and SerDe operations ========================= */ -async fn get_db_from_file(target_folder: &str) -> Result { +pub async fn get_db_from_file(target_folder: &str) -> Result { let urls: UrlSet = serde_json::de::from_slice(&std::fs::read(format!("{target_folder}/{DB_FILE_NAME}"))?) .unwrap(); diff --git a/sitemap-generator/src/sitemap/mod.rs b/sitemap-generator/src/sitemap/mod.rs index 21200bd..31ba461 100644 --- a/sitemap-generator/src/sitemap/mod.rs +++ b/sitemap-generator/src/sitemap/mod.rs @@ -5,6 +5,7 @@ use std::ops::{Deref, DerefMut}; use serde::{Deserialize, Serialize}; use tinytemplate::TinyTemplate; +use tracing::debug; use crate::app::SitemapConfig; @@ -57,10 +58,22 @@ impl UrlSet { pub fn find_affected(&mut self, id: &str, slug: &str) -> Vec<&mut Url> { self.iter_mut() .filter(|u| { - u.data.id == id && u.data.slug != slug - || u.related + debug!( + "comparing: ( {} == {} && {} != {} ) || ( {:?} == {} && {:?} != {} )", + &u.data.id, + &id, + &u.data.slug, + &slug, + u.related.clone().map(|ud| ud.id), + &id, + u.related.clone().map(|ud| ud.slug), + &slug + ); + (u.data.id == id && u.data.slug != slug) + || (u + .related .as_ref() - .map_or(false, |ud| ud.id == id && ud.slug != slug) + .map_or(false, |ud| ud.id == id && ud.slug != slug)) }) .collect() } diff --git a/sitemap-generator/src/tests/mod.rs b/sitemap-generator/src/tests/mod.rs index 375b8ce..dc928a6 100644 --- a/sitemap-generator/src/tests/mod.rs +++ b/sitemap-generator/src/tests/mod.rs @@ -144,6 +144,7 @@ async fn sequence_of_actions_is_preserved() { #[rstest] #[traced_test] +#[parallel] fn urlset_serialisation_isnt_lossy() { std::env::set_var("APP_API_BASE_URL", "http://localhost:3000"); let (_, sitemap_config) = testing_configs(); @@ -156,3 +157,17 @@ fn urlset_serialisation_isnt_lossy() { let deserialized_url_set: UrlSet = serde_cbor::de::from_slice(&file_str).unwrap(); assert_eq!(url_set, deserialized_url_set); } +//TODO: TEST UPDATES AND DELETES, UPDATING URL CREATES A NEW ENTRY INSTEAD OF EDITING PREVIOUS ONE + +// #[rstest] +// #[traced_test] +// #[parallel] +// async fn url_set_find_affected_works() { +// let mut url_set = get_db_from_file("./").await.unwrap(); +// assert!(url_set +// .find_affected("UHJvZHVjdDoxNjEwMg==", "dute-vlakno-0-5kg-biele") +// .is_empty()); +// assert!(!url_set +// .find_affected("UHJvZHVjdDoxNjEwMg==", "dute-vlakno-0-5kg-biele-test") +// .is_empty()); +// } diff --git a/sitemap-generator/src/tests/utils.rs b/sitemap-generator/src/tests/utils.rs index 2b41e5c..297af05 100644 --- a/sitemap-generator/src/tests/utils.rs +++ b/sitemap-generator/src/tests/utils.rs @@ -109,211 +109,115 @@ impl Distribution for Standard { // action_type = rand::random::(); // } // -// match rand::random::() { -// ItemType::Product => { -// // If there is a category url already, use that for relation instead of always a -// let mut is_using_existing_category = false; -// // new one -// if res -// .iter() -// .find(|r| r.url.data.typ == ItemType::Category) -// .is_some() -// { -// match rand::random::() { -// true => loop { -// let r = res.choose(&mut rand::thread_rng()).unwrap().clone(); -// if r.url.data.typ == ItemType::Category { -// rel_slug = r.url.data.slug; -// rel_id = cynic::Id::new(r.url.data.id); -// is_using_existing_category = true; -// break; -// } -// }, -// false => (), -// }; -// } -// let product_updated: String = match action_type { -// ActionType::Create => serde_json::to_string_pretty(&ProductCreated { -// product: Some(Product { -// id: id.clone(), -// slug: slug.clone(), -// category: Some(Category { -// slug: rel_slug.clone(), -// id: rel_id.clone(), -// }), -// }), -// }) -// .unwrap(), -// ActionType::Update => { -// let p; -// loop { -// let c = res.choose(&mut rand::thread_rng()).unwrap().clone(); -// if c.action_type != ActionType::Delete { -// p = c; -// break; -// } -// } -// serde_json::to_string_pretty(&ProductUpdated { -// product: Some(Product { -// id: cynic::Id::new(p.url.data.id), -// slug: p.url.data.slug.clone(), -// category: p.url.related.map(|c| Category { -// slug: c.slug.clone(), -// id: cynic::Id::new(c.id), -// }), -// }), -// }) +// let item_type = rand::random::(); +// // If there is a category url already, use that for relation instead of always a +// let mut is_using_existing_category = false; +// // new one +// if res +// .iter() +// .find(|r| r.url.data.typ == ItemType::Category) +// .is_some() +// { +// match rand::random::() { +// true => loop { +// let r = res.choose(&mut rand::thread_rng()).unwrap().clone(); +// if r.url.data.typ == ItemType::Category { +// rel_slug = r.url.data.slug; +// rel_id = cynic::Id::new(r.url.data.id); +// is_using_existing_category = true; +// break; // } -// .unwrap(), -// ActionType::Delete => { -// let p; -// loop { -// let c = res.choose(&mut rand::thread_rng()).unwrap().clone(); -// if c.action_type != ActionType::Delete { -// p = c; -// break; -// } -// } -// serde_json::to_string_pretty(&ProductUpdated { -// product: Some(Product { -// id: id.clone(), -// slug: slug.clone(), -// category: Some(Category { -// slug: rel_slug.clone(), -// id: rel_id.clone(), -// }), -// }), -// }) -// .unwrap()} -// }; -// let url = Url::new( -// product_updated.clone(), -// &sitemap_config, -// ItemData { -// id: id.clone().inner().to_owned(), -// slug: slug.clone(), -// typ: ItemType::Product, -// }, -// Some(ItemData { -// id: rel_id.inner().to_owned(), -// slug: rel_slug.clone(), -// typ: ItemType::Category, -// }), -// ) -// .unwrap(); -// -// if !is_using_existing_category { -// let category_updated = CategoryUpdated { -// category: Some(Category2 { -// id: rel_id.clone(), -// slug: rel_slug.clone(), -// }), -// }; -// -// let cat_url = Url::new( -// category_updated.clone(), -// &sitemap_config, -// ItemData { -// id: id.clone().inner().to_owned(), -// slug: slug.clone(), -// typ: ItemType::Category, -// }, -// None, -// ) -// .unwrap(); -// res.push(( -// serde_json::to_string_pretty(&category_updated).unwrap(), -// cat_url, -// EitherWebhookType::Async(AsyncWebhookEventType::CategoryCreated), -// )); -// } -// -// res.push(( -// serde_json::to_string_pretty(&product_updated).unwrap(), -// url, -// EitherWebhookType::Async(AsyncWebhookEventType::ProductCreated), -// )); -// } -// ItemType::Category => { -// let category_updated = CategoryUpdated { -// category: Some(Category2 { -// id: id.clone(), -// slug: slug.clone(), -// }), -// }; -// -// let url = Url::new( -// category_updated.clone(), -// &sitemap_config, -// ItemData { -// id: id.clone().inner().to_owned(), -// slug: slug.clone(), -// typ: ItemType::Category, -// }, -// None, -// ) -// .unwrap(); -// res.push(( -// serde_json::to_string_pretty(&category_updated).unwrap(), -// url, -// EitherWebhookType::Async(AsyncWebhookEventType::CategoryCreated), -// )); -// } -// ItemType::Collection => { -// let collection_updated = CollectionUpdated { -// collection: Some(Collection { -// id: id.clone(), -// slug: slug.clone(), -// }), -// }; -// -// let url = Url::new( -// collection_updated.clone(), -// &sitemap_config, -// ItemData { -// id: id.clone().inner().to_owned(), -// slug: slug.clone(), -// typ: ItemType::Collection, -// }, -// None, -// ) -// .unwrap(); -// res.push(( -// serde_json::to_string_pretty(&collection_updated).unwrap(), -// url, -// EitherWebhookType::Async(AsyncWebhookEventType::CollectionCreated), -// )); -// } -// ItemType::Page => { -// let page_updated = PageUpdated { -// page: Some(Page { -// id: id.clone(), -// slug: slug.clone(), -// }), -// }; -// -// let url = Url::new( -// page_updated.clone(), -// &sitemap_config, -// ItemData { -// id: id.clone().inner().to_owned(), -// slug: slug.clone(), -// typ: ItemType::Page, -// }, -// None, -// ) -// .unwrap(); -// res.push(( -// serde_json::to_string_pretty(&page_updated).unwrap(), -// url, -// EitherWebhookType::Async(AsyncWebhookEventType::PageCreated), -// )); -// } +// }, +// false => (), +// }; // } +// let body_data: String = match (action_type, item_type) { +// (ActionType::Create, ItemType::Product) => { +// serde_json::to_string_pretty(&ProductCreated { +// product: Some(Product { +// id: id.clone(), +// slug: slug.clone(), +// category: Some(Category { +// slug: rel_slug.clone(), +// id: rel_id.clone(), +// }), +// }), +// }) +// .unwrap() +// } +// (ActionType::Update, ItemType::Product) => { +// let p; +// loop { +// let c = res.choose(&mut rand::thread_rng()).unwrap().clone(); +// if c.action_type != ActionType::Delete { +// p = c; +// break; +// } +// } +// serde_json::to_string_pretty(&ProductUpdated { +// product: Some(Product { +// id: cynic::Id::new(p.url.data.id), +// slug: p.url.data.slug.clone(), +// category: p.url.related.map(|c| Category { +// slug: c.slug.clone(), +// id: cynic::Id::new(c.id), +// }), +// }), +// }) +// } +// (ActionType::Delete, ) => {} +// }; +// let url = Url::new( +// product_updated.clone(), +// &sitemap_config, +// ItemData { +// id: id.clone().inner().to_owned(), +// slug: slug.clone(), +// typ: ItemType::Product, +// }, +// Some(ItemData { +// id: rel_id.inner().to_owned(), +// slug: rel_slug.clone(), +// typ: ItemType::Category, +// }), +// ) +// .unwrap(); +// +// if !is_using_existing_category { +// let category_updated = CategoryUpdated { +// category: Some(Category2 { +// id: rel_id.clone(), +// slug: rel_slug.clone(), +// }), +// }; +// +// let cat_url = Url::new( +// category_updated.clone(), +// &sitemap_config, +// ItemData { +// id: id.clone().inner().to_owned(), +// slug: slug.clone(), +// typ: ItemType::Category, +// }, +// None, +// ) +// .unwrap(); +// res.push(( +// serde_json::to_string_pretty(&category_updated).unwrap(), +// cat_url, +// EitherWebhookType::Async(AsyncWebhookEventType::CategoryCreated), +// )); +// } +// +// res.push(( +// serde_json::to_string_pretty(&product_updated).unwrap(), +// url, +// EitherWebhookType::Async(AsyncWebhookEventType::ProductCreated), +// )); // } // res // } - +// pub fn gen_random_url_set( len: usize, sitemap_config: &SitemapConfig, @@ -331,10 +235,7 @@ pub fn gen_random_url_set( // If there is a category url already, use that for relation instead of always a let mut is_using_existing_category = false; // new one - if res - .iter() - .any(|r| r.1.data.typ == ItemType::Category) - { + if res.iter().any(|r| r.1.data.typ == ItemType::Category) { match rand::random::() { true => loop { let r = res.choose(&mut rand::thread_rng()).unwrap().clone(); @@ -347,7 +248,7 @@ pub fn gen_random_url_set( }, false => (), }; - } + } let product_updated = ProductUpdated { product: Some(Product { id: id.clone(),