From 049aa6f2b14e95be352147cf7e221b4e6c82bcfe Mon Sep 17 00:00:00 2001
From: mehbark <terezi@pyrope.net>
Date: Thu, 27 Mar 2025 02:36:22 -0400
Subject: [PATCH] feat: one transaction, build in memory and vacuum out
---
src/main.rs | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index 0e90462..1f0b1bf 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -45,11 +45,13 @@ fn run(
);
log::info!("opening connection");
- let mut conn = Connection::open(output_file)?;
+ let mut conn = Connection::open_in_memory()?;
create_tables(&mut conn)?;
insert_stats(&mut conn, &stats)?;
+ conn.execute("VACUUM INTO ?1", (output_file,))?;
+
Ok(())
}
@@ -58,8 +60,9 @@ fn run(
struct Config {
#[arg()]
input_file: PathBuf,
+ // TODO: non-utf-8 (the problem is saving the database)
#[arg(short, long = "output")]
- output_file: PathBuf,
+ output_file: String,
}
#[derive(Error, Debug)]
@@ -125,12 +128,12 @@ fn insert_stats(conn: &mut Connection, stats: &Stats) -> rusqlite::Result<()> {
let mon_count = stats.data.len();
let mon_count_digits = mon_count.to_string().len();
+ let tx = conn.transaction()?;
for (i, (mon, data)) in stats.data.iter().enumerate() {
log::debug!(
"Processing mon {:mon_count_digits$}/{mon_count} ({mon})",
i + 1
);
- let tx = conn.transaction()?;
// normalizing with mon_count gives us data that is much easier to work
// with. for example, if pikachu has 10k count (weighted) and thunderbolt
@@ -196,9 +199,8 @@ fn insert_stats(conn: &mut Connection, stats: &Stats) -> rusqlite::Result<()> {
(mon, opp, percentage, stddev),
)?;
}
-
- tx.commit()?;
}
+ tx.commit()?;
Ok(())
}