djkato/CSQL
1
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

auto parse and match csv to fields

Browse source
This commit is contained in:
Djkato 2023-03-25 00:04:12 +01:00
parent de878ea6f6
commit bcda74ff35
4 changed files with 143 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

81
src/backend/backend_manager.rs
View file

@ -1,5 +1,5 @@
use super::csv_handler::ImportedData; use super::csv_handler::{DataEntry, ImportedData};
use super::database_handler::{DBLoginData, Tables}; use super::database_handler::{DBLoginData, TableField, Tables};
use super::parser::parse; use super::parser::parse;
use sqlx::MySqlConnection; use sqlx::MySqlConnection;
use std::sync::Arc; use std::sync::Arc;
@ -70,22 +70,69 @@ impl BackendManger {
} }
} }
} }
Communication::GetTableDescription(table_index) => { Communication::GetTableDescription(table_index, sender) => {
let mut db_table_data = self.db_table_data.lock().await; let mut db_table_data = self.db_table_data.lock().await;
let mut csv_data = self.csv_data.lock().await;
db_table_data db_table_data
.tables .tables
.get_mut(table_index) .get_mut(table_index)
.unwrap() .unwrap()
.describe_table(&mut self.db_connection.as_mut().unwrap()) .describe_table(&mut self.db_connection.as_mut().unwrap())
.await; .await;
if csv_data.are_headers {
match try_match_headers_to_fields(
&mut db_table_data
.tables
.get_mut(table_index)
.unwrap()
.fields
.as_mut()
.unwrap()
.as_mut(),
&csv_data.data.iter_row(0),
) {
Ok(_) => {
for field in db_table_data
.tables
.get_mut(table_index)
.unwrap()
.fields
.as_mut()
.unwrap()
{
println!(
" > automapping field \"{}\" to col \"{:?}\"",
&field.description.field, field.mapped_to_col
);
if let Some(col_index) = field.mapped_to_col {
for cell in csv_data.data.iter_col_mut(col_index) {
cell.curr_field_description =
Some(field.description.clone());
parse(cell);
}
}
}
sender.send(true).unwrap_or_else(|e| {
println!(
"Server - failed to respond to getTableDescription, {}",
e
)
});
}
Err(_) => sender.send(false).unwrap_or_else(|e| {
println!("Server - failed to respond to getTableDescription, {}", e)
}),
}
} else {
sender.send(false).unwrap_or_else(|e| {
println!("Server - failed to respond to getTableDescription, {}", e)
});
}
} }
Communication::TryParseCol(col_index) => { Communication::TryParseCol(col_index) => {
let mut csv_data = self.csv_data.lock().await; let mut csv_data = self.csv_data.lock().await;
for cell in csv_data.data.iter_col_mut(col_index) { for cell in csv_data.data.iter_col_mut(col_index) {
if cell.curr_field_description.is_some() { if cell.curr_field_description.is_some() {
if let Some(desc) = &cell.curr_field_description {
println!("{} should be of type {}", cell.data, desc.field_type)
}
parse(cell); parse(cell);
} }
} }
@ -94,6 +141,26 @@ impl BackendManger {
} }
} }
} }
pub fn try_match_headers_to_fields(
db_fields: &mut Vec<TableField>,
csv_headers: &std::slice::Iter<'_, DataEntry>,
) -> Result<(), ()> {
let mut has_matched_some = false;
for field in db_fields {
for (i, header) in csv_headers.clone().enumerate() {
if &field.description.field == &header.data {
field.mapped_to_col = Some(i);
has_matched_some = true;
}
}
}
match has_matched_some {
true => return Ok(()),
false => return Err(()),
}
}
pub enum Communication { pub enum Communication {
ValidateCreditentials( ValidateCreditentials(
DBLoginData, DBLoginData,
@ -101,6 +168,6 @@ pub enum Communication {
), ),
AreCreditentialsValidated(oneshot::Sender<bool>), AreCreditentialsValidated(oneshot::Sender<bool>),
LoadImportFilePath(String, oneshot::Sender<bool>), LoadImportFilePath(String, oneshot::Sender<bool>),
GetTableDescription(usize), GetTableDescription(usize, oneshot::Sender<bool>),
TryParseCol(usize), TryParseCol(usize),
} }

4
src/backend/database_handler.rs
View file

@ -8,12 +8,12 @@ pub struct Tables {
pub tables: Vec<Table>, pub tables: Vec<Table>,
} }
#[derive(Default)] #[derive(Default, Clone)]
pub struct Table { pub struct Table {
pub name: String, pub name: String,
pub fields: Option<Vec<TableField>>, pub fields: Option<Vec<TableField>>,
} }
#[derive(Default)] #[derive(Default, Clone)]
pub struct TableField { pub struct TableField {
pub description: FieldDescription, pub description: FieldDescription,
pub mapped_to_col: Option<usize>, pub mapped_to_col: Option<usize>,

48
src/backend/parser.rs
View file

@ -202,30 +202,38 @@ fn parse_tinyint(cell: &mut DataEntry, args: Option<String>) {
} }
} }
fn parse_date(cell: &mut DataEntry) { fn parse_date(cell: &mut DataEntry) {
//YYYY-MM-DD from ‘1000-01-01’ to ‘9999-12-31’ //YYYY-MM-DD from ‘1000-01-01’ to ‘9999-12-31’,also ’0000-00-00’
match NaiveDate::parse_from_str(&cell.data, "%Y-%m-%d") { if cell.data == "0000-00-00" {
Ok(_) => cell.is_parsed = Some(Ok(())), cell.is_parsed = Some(Ok(()))
Err(e) => { } else {
cell.is_parsed = Some(Err(Arc::from(<String as Into< match NaiveDate::parse_from_str(&cell.data, "%Y-%m-%d") {
Box<dyn Error + Send + Sync>, Ok(_) => cell.is_parsed = Some(Ok(())),
>>::into(format!( Err(e) => {
"Invalid date: {}", cell.is_parsed = Some(Err(Arc::from(<String as Into<
e Box<dyn Error + Send + Sync>,
))))) >>::into(format!(
"Invalid date: {}",
e
)))))
}
} }
} }
} }
fn parse_datetime(cell: &mut DataEntry) { fn parse_datetime(cell: &mut DataEntry) {
//YYYY-MM-DD HH:MM:SS //YYYY-MM-DD HH:MM:SS, also ’0000-00-00’
match NaiveDateTime::parse_from_str(&cell.data, "%Y-%m-%d %H:%M:%S") { if cell.data == "0000-00-00 00:00:00" {
Ok(_) => cell.is_parsed = Some(Ok(())), cell.is_parsed = Some(Ok(()))
Err(e) => { } else {
cell.is_parsed = Some(Err(Arc::from(<String as Into< match NaiveDateTime::parse_from_str(&cell.data, "%Y-%m-%d %H:%M:%S") {
Box<dyn Error + Send + Sync>, Ok(_) => cell.is_parsed = Some(Ok(())),
>>::into(format!( Err(e) => {
"Invalid date: {}", cell.is_parsed = Some(Err(Arc::from(<String as Into<
e Box<dyn Error + Send + Sync>,
))))) >>::into(format!(
"Invalid date: {}",
e
)))))
}
} }
} }
} }

72
src/ui/table_window.rs
View file

@ -1,7 +1,6 @@
use crate::backend::backend_manager::Communication; use crate::backend::backend_manager::Communication;
use crate::backend::csv_handler::ImportedData; use crate::backend::csv_handler::ImportedData;
use crate::backend::database_handler::Tables; use crate::backend::database_handler::{Tables,TableField};
use eframe::glow::CONSTANT_COLOR;
use egui::{ComboBox, Context, Ui}; use egui::{ComboBox, Context, Ui};
use egui_extras::{Column, Size, StripBuilder, TableBuilder}; use egui_extras::{Column, Size, StripBuilder, TableBuilder};
use std::error::Error; use std::error::Error;
@ -15,7 +14,10 @@ pub struct SpreadSheetWindow {
is_file_loaded_receiver: Option<oneshot::Receiver<bool>>, is_file_loaded_receiver: Option<oneshot::Receiver<bool>>,
is_file_loaded: bool, is_file_loaded: bool,
current_table: Option<usize>, current_table: Option<usize>,
is_current_table_described: bool, is_table_described_receiver: Option<oneshot::Receiver<bool>>,
is_table_described: bool,
is_table_matched_with_headers: bool,
} }
impl SpreadSheetWindow { impl SpreadSheetWindow {
pub fn default( pub fn default(
@ -27,10 +29,12 @@ impl SpreadSheetWindow {
sender, sender,
is_file_loaded_receiver: None, is_file_loaded_receiver: None,
is_file_loaded: false, is_file_loaded: false,
is_table_described_receiver: None,
is_table_described: false,
is_table_matched_with_headers: false,
csv_data_handle, csv_data_handle,
db_table_data_handle, db_table_data_handle,
current_table: None, current_table: None,
is_current_table_described: false,
} }
} }
} }
@ -122,23 +126,36 @@ impl SpreadSheetWindow {
}); });
/* If a table is selected, try if it's fields are discovered */ /* If a table is selected, try if it's fields are discovered */
if let Some(table_i) = self.current_table { if !self.is_table_described{
if db_table_data.tables.get(table_i).unwrap().fields.is_some() {
self.is_current_table_described = true; if let Some(table_i) = self.current_table {
} else { if db_table_data.tables.get(table_i).unwrap().fields.is_some() {
self.is_table_described = true;
} else if self.is_table_described_receiver.is_some(){
match self.is_table_described_receiver.as_mut().unwrap().try_recv(){
Ok(res) => {self.is_table_matched_with_headers = res},
Err(e) => println!("Failed receiving is_table_described, {}",e),
}
} else{
let (sender, receiver ) = oneshot::channel();
self.is_table_described_receiver = Some(receiver);
self.sender self.sender
.try_send(Communication::GetTableDescription(table_i)) .try_send(Communication::GetTableDescription(table_i, sender))
.unwrap_or_else(|e| println!("Failed asking to describe table, {}", e)); .unwrap_or_else(|e| println!("Failed asking to describe table, {}", e));
}
} }
} }
if self.is_current_table_described { }
}
if self.is_table_described {
self.table_builder(ui); self.table_builder(ui);
} }
} }
fn table_builder(&mut self, ui: &mut Ui) { fn table_builder(&mut self, ui: &mut Ui) {
if let Ok(csv_data) = &mut self.csv_data_handle.try_lock() { if let Ok(csv_data) = &mut self.csv_data_handle.try_lock() {
if let Ok(db_table_data) = &mut self.db_table_data_handle.try_lock() { if let Ok(db_table_data) = &mut self.db_table_data_handle.try_lock() {
//ref to all fields in curr db table
let mut curr_db_table_fields: &mut Vec<TableField> = db_table_data.tables.get_mut(self.current_table.unwrap()).unwrap().fields.as_mut().unwrap();
let mut table = TableBuilder::new(ui) let mut table = TableBuilder::new(ui)
.striped(true) .striped(true)
.cell_layout(egui::Layout::left_to_right(egui::Align::Center)); .cell_layout(egui::Layout::left_to_right(egui::Align::Center));
@ -146,7 +163,8 @@ impl SpreadSheetWindow {
for _i in 0..csv_data.data.cols() { for _i in 0..csv_data.data.cols() {
table = table.column(Column::auto().resizable(true).clip(false)); table = table.column(Column::auto().resizable(true).clip(false));
} }
/* If cols got prematched by name */
table table
.column(Column::remainder()) .column(Column::remainder())
.min_scrolled_height(0.0) .min_scrolled_height(0.0)
@ -162,33 +180,21 @@ impl SpreadSheetWindow {
} else { } else {
combo_box = ComboBox::new(i, ""); combo_box = ComboBox::new(i, "");
} }
//if any field is assinged to this combobox, show it's text, else "----" //if any field is assinged to this combobox, show it's text, else "----"
if let Some(selected_field) = db_table_data if let Some(selected_field) = curr_db_table_fields
.tables
.get(self.current_table.unwrap())
.unwrap()
.fields
.as_ref()
.unwrap()
.iter() .iter()
.find(|field| field.mapped_to_col == Some(i)) .find(|field| field.mapped_to_col == Some(i))
{ {
combo_box = combo_box combo_box = combo_box
.selected_text(selected_field.description.field.clone()); .selected_text(selected_field.description.field.clone());
} else { } else{
combo_box = combo_box.selected_text("-----"); combo_box = combo_box.selected_text("-----");
} }
/* When a Field gets attached to Col, */ /* When a Field gets attached to Col, */
combo_box.show_ui(ui, |ui| { combo_box.show_ui(ui, |ui| {
for field in db_table_data for field in curr_db_table_fields.iter_mut()
.tables
.get_mut(self.current_table.unwrap())
.unwrap()
.fields
.as_mut()
.unwrap()
.iter_mut()
{ {
if ui if ui
.selectable_value( .selectable_value(
@ -198,7 +204,7 @@ impl SpreadSheetWindow {
) )
.clicked() .clicked()
{ {
self.is_current_table_described = false; self.is_table_described = false;
match self.sender.try_send(Communication::TryParseCol(i)){ match self.sender.try_send(Communication::TryParseCol(i)){
Ok(_) => { Ok(_) => {
for cel in csv_data.data.iter_col_mut(i) { for cel in csv_data.data.iter_col_mut(i) {
@ -215,9 +221,9 @@ impl SpreadSheetWindow {
} }
}) })
.body(|body| { .body(|body| {
body.rows(15., csv_data.data.rows(), |row_index, mut row| { body.rows(15., csv_data.data.rows() -1, |row_index, mut row| {
for curr_cell in for curr_cell in
csv_data.data.iter_row_mut(row_index) csv_data.data.iter_row_mut(row_index+1)
{ {
/* If cell is bound to a field, color it's bg according to is_parsed */ /* If cell is bound to a field, color it's bg according to is_parsed */
row.col(|ui| { row.col(|ui| {