attempt to fix UrlUpdates creating new urls instead of updating
This commit is contained in:
djkato
parent
66ea84db86
commit
d199d679c4
5 changed files with 142 additions and 213 deletions
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "sitemap-generator"
|
||||
version = "1.0.0"
|
||||
version = "1.0.1"
|
||||
edition = "2021"
|
||||
authors = ["Djkáťo <djkatovfx@gmail.com>"]
|
||||
description = "Creates and keeps Sitemap.xml uptodate with Saleor."
|
||||
|
|
|
|||
|
|
@ -16,14 +16,14 @@ use crate::{
|
|||
sitemap::Url,
|
||||
};
|
||||
use tokio::{sync::mpsc::Receiver, task::JoinHandle};
|
||||
use tracing::{debug, error, info, trace, warn};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
use super::{ItemData, ItemType, UrlSet};
|
||||
|
||||
// 10k links google says, but there's also a size limit and my custom params might be messing with
|
||||
// that? Rather split prematurely to be sure.
|
||||
const MAX_URL_IN_SET: usize = 50_000;
|
||||
const DB_FILE_NAME: &str = "db.toml";
|
||||
const DB_FILE_NAME: &str = "db.json";
|
||||
const SITEMAP_FILE_NAME: &str = "sitemap.txt";
|
||||
|
||||
pub struct EventHandler {
|
||||
|
|
@ -297,7 +297,7 @@ async fn update_or_create<T: Serialize>(
|
|||
debug!("affected urls: {:?}", &affected_urls);
|
||||
|
||||
if affected_urls.is_empty() {
|
||||
trace!("{:?} doesn't exist in url_set yet", &item.slug);
|
||||
debug!("{:?} doesn't exist in url_set yet", &item.slug);
|
||||
url_set.push(Url::new(data, sitemap_config, item, rel_item).unwrap());
|
||||
} else {
|
||||
// Update affected urls
|
||||
|
|
@ -353,7 +353,7 @@ async fn delete(id: &str, sitemap_config: &SitemapConfig) {
|
|||
|
||||
/* =================== File and SerDe operations ========================= */
|
||||
|
||||
async fn get_db_from_file(target_folder: &str) -> Result<UrlSet, UrlSetFileOperationsErr> {
|
||||
pub async fn get_db_from_file(target_folder: &str) -> Result<UrlSet, UrlSetFileOperationsErr> {
|
||||
let urls: UrlSet =
|
||||
serde_json::de::from_slice(&std::fs::read(format!("{target_folder}/{DB_FILE_NAME}"))?)
|
||||
.unwrap();
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ use std::ops::{Deref, DerefMut};
|
|||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tinytemplate::TinyTemplate;
|
||||
use tracing::debug;
|
||||
|
||||
use crate::app::SitemapConfig;
|
||||
|
||||
|
|
@ -57,10 +58,22 @@ impl UrlSet {
|
|||
pub fn find_affected(&mut self, id: &str, slug: &str) -> Vec<&mut Url> {
|
||||
self.iter_mut()
|
||||
.filter(|u| {
|
||||
u.data.id == id && u.data.slug != slug
|
||||
|| u.related
|
||||
debug!(
|
||||
"comparing: ( {} == {} && {} != {} ) || ( {:?} == {} && {:?} != {} )",
|
||||
&u.data.id,
|
||||
&id,
|
||||
&u.data.slug,
|
||||
&slug,
|
||||
u.related.clone().map(|ud| ud.id),
|
||||
&id,
|
||||
u.related.clone().map(|ud| ud.slug),
|
||||
&slug
|
||||
);
|
||||
(u.data.id == id && u.data.slug != slug)
|
||||
|| (u
|
||||
.related
|
||||
.as_ref()
|
||||
.map_or(false, |ud| ud.id == id && ud.slug != slug)
|
||||
.map_or(false, |ud| ud.id == id && ud.slug != slug))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -144,6 +144,7 @@ async fn sequence_of_actions_is_preserved() {
|
|||
|
||||
#[rstest]
|
||||
#[traced_test]
|
||||
#[parallel]
|
||||
fn urlset_serialisation_isnt_lossy() {
|
||||
std::env::set_var("APP_API_BASE_URL", "http://localhost:3000");
|
||||
let (_, sitemap_config) = testing_configs();
|
||||
|
|
@ -156,3 +157,17 @@ fn urlset_serialisation_isnt_lossy() {
|
|||
let deserialized_url_set: UrlSet = serde_cbor::de::from_slice(&file_str).unwrap();
|
||||
assert_eq!(url_set, deserialized_url_set);
|
||||
}
|
||||
//TODO: TEST UPDATES AND DELETES, UPDATING URL CREATES A NEW ENTRY INSTEAD OF EDITING PREVIOUS ONE
|
||||
|
||||
// #[rstest]
|
||||
// #[traced_test]
|
||||
// #[parallel]
|
||||
// async fn url_set_find_affected_works() {
|
||||
// let mut url_set = get_db_from_file("./").await.unwrap();
|
||||
// assert!(url_set
|
||||
// .find_affected("UHJvZHVjdDoxNjEwMg==", "dute-vlakno-0-5kg-biele")
|
||||
// .is_empty());
|
||||
// assert!(!url_set
|
||||
// .find_affected("UHJvZHVjdDoxNjEwMg==", "dute-vlakno-0-5kg-biele-test")
|
||||
// .is_empty());
|
||||
// }
|
||||
|
|
|
|||
|
|
@ -109,8 +109,7 @@ impl Distribution<ActionType> for Standard {
|
|||
// action_type = rand::random::<ActionType>();
|
||||
// }
|
||||
//
|
||||
// match rand::random::<ItemType>() {
|
||||
// ItemType::Product => {
|
||||
// let item_type = rand::random::<ItemType>();
|
||||
// // If there is a category url already, use that for relation instead of always a
|
||||
// let mut is_using_existing_category = false;
|
||||
// // new one
|
||||
|
|
@ -132,8 +131,9 @@ impl Distribution<ActionType> for Standard {
|
|||
// false => (),
|
||||
// };
|
||||
// }
|
||||
// let product_updated: String = match action_type {
|
||||
// ActionType::Create => serde_json::to_string_pretty(&ProductCreated {
|
||||
// let body_data: String = match (action_type, item_type) {
|
||||
// (ActionType::Create, ItemType::Product) => {
|
||||
// serde_json::to_string_pretty(&ProductCreated {
|
||||
// product: Some(Product {
|
||||
// id: id.clone(),
|
||||
// slug: slug.clone(),
|
||||
|
|
@ -143,8 +143,9 @@ impl Distribution<ActionType> for Standard {
|
|||
// }),
|
||||
// }),
|
||||
// })
|
||||
// .unwrap(),
|
||||
// ActionType::Update => {
|
||||
// .unwrap()
|
||||
// }
|
||||
// (ActionType::Update, ItemType::Product) => {
|
||||
// let p;
|
||||
// loop {
|
||||
// let c = res.choose(&mut rand::thread_rng()).unwrap().clone();
|
||||
|
|
@ -164,27 +165,7 @@ impl Distribution<ActionType> for Standard {
|
|||
// }),
|
||||
// })
|
||||
// }
|
||||
// .unwrap(),
|
||||
// ActionType::Delete => {
|
||||
// let p;
|
||||
// loop {
|
||||
// let c = res.choose(&mut rand::thread_rng()).unwrap().clone();
|
||||
// if c.action_type != ActionType::Delete {
|
||||
// p = c;
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// serde_json::to_string_pretty(&ProductUpdated {
|
||||
// product: Some(Product {
|
||||
// id: id.clone(),
|
||||
// slug: slug.clone(),
|
||||
// category: Some(Category {
|
||||
// slug: rel_slug.clone(),
|
||||
// id: rel_id.clone(),
|
||||
// }),
|
||||
// }),
|
||||
// })
|
||||
// .unwrap()}
|
||||
// (ActionType::Delete, ) => {}
|
||||
// };
|
||||
// let url = Url::new(
|
||||
// product_updated.clone(),
|
||||
|
|
@ -234,86 +215,9 @@ impl Distribution<ActionType> for Standard {
|
|||
// EitherWebhookType::Async(AsyncWebhookEventType::ProductCreated),
|
||||
// ));
|
||||
// }
|
||||
// ItemType::Category => {
|
||||
// let category_updated = CategoryUpdated {
|
||||
// category: Some(Category2 {
|
||||
// id: id.clone(),
|
||||
// slug: slug.clone(),
|
||||
// }),
|
||||
// };
|
||||
//
|
||||
// let url = Url::new(
|
||||
// category_updated.clone(),
|
||||
// &sitemap_config,
|
||||
// ItemData {
|
||||
// id: id.clone().inner().to_owned(),
|
||||
// slug: slug.clone(),
|
||||
// typ: ItemType::Category,
|
||||
// },
|
||||
// None,
|
||||
// )
|
||||
// .unwrap();
|
||||
// res.push((
|
||||
// serde_json::to_string_pretty(&category_updated).unwrap(),
|
||||
// url,
|
||||
// EitherWebhookType::Async(AsyncWebhookEventType::CategoryCreated),
|
||||
// ));
|
||||
// }
|
||||
// ItemType::Collection => {
|
||||
// let collection_updated = CollectionUpdated {
|
||||
// collection: Some(Collection {
|
||||
// id: id.clone(),
|
||||
// slug: slug.clone(),
|
||||
// }),
|
||||
// };
|
||||
//
|
||||
// let url = Url::new(
|
||||
// collection_updated.clone(),
|
||||
// &sitemap_config,
|
||||
// ItemData {
|
||||
// id: id.clone().inner().to_owned(),
|
||||
// slug: slug.clone(),
|
||||
// typ: ItemType::Collection,
|
||||
// },
|
||||
// None,
|
||||
// )
|
||||
// .unwrap();
|
||||
// res.push((
|
||||
// serde_json::to_string_pretty(&collection_updated).unwrap(),
|
||||
// url,
|
||||
// EitherWebhookType::Async(AsyncWebhookEventType::CollectionCreated),
|
||||
// ));
|
||||
// }
|
||||
// ItemType::Page => {
|
||||
// let page_updated = PageUpdated {
|
||||
// page: Some(Page {
|
||||
// id: id.clone(),
|
||||
// slug: slug.clone(),
|
||||
// }),
|
||||
// };
|
||||
//
|
||||
// let url = Url::new(
|
||||
// page_updated.clone(),
|
||||
// &sitemap_config,
|
||||
// ItemData {
|
||||
// id: id.clone().inner().to_owned(),
|
||||
// slug: slug.clone(),
|
||||
// typ: ItemType::Page,
|
||||
// },
|
||||
// None,
|
||||
// )
|
||||
// .unwrap();
|
||||
// res.push((
|
||||
// serde_json::to_string_pretty(&page_updated).unwrap(),
|
||||
// url,
|
||||
// EitherWebhookType::Async(AsyncWebhookEventType::PageCreated),
|
||||
// ));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// res
|
||||
// }
|
||||
|
||||
//
|
||||
pub fn gen_random_url_set(
|
||||
len: usize,
|
||||
sitemap_config: &SitemapConfig,
|
||||
|
|
@ -331,10 +235,7 @@ pub fn gen_random_url_set(
|
|||
// If there is a category url already, use that for relation instead of always a
|
||||
let mut is_using_existing_category = false;
|
||||
// new one
|
||||
if res
|
||||
.iter()
|
||||
.any(|r| r.1.data.typ == ItemType::Category)
|
||||
{
|
||||
if res.iter().any(|r| r.1.data.typ == ItemType::Category) {
|
||||
match rand::random::<bool>() {
|
||||
true => loop {
|
||||
let r = res.choose(&mut rand::thread_rng()).unwrap().clone();
|
||||
|
|
|
|||
Loading…
Reference in a new issue