From 8bf355f9563c5680c642b56973277c8fb6300a47 Mon Sep 17 00:00:00 2001 From: appflowy Date: Fri, 9 Jul 2021 16:34:50 +0800 Subject: [PATCH] use diesel as sqlite orm & setup db init schedule --- .idea/appflowy_client.iml | 2 + rust-lib/Cargo.toml | 2 + rust-lib/flowy-db/.env | 1 + rust-lib/flowy-db/Cargo.toml | 12 ++ rust-lib/flowy-db/diesel.toml | 5 + rust-lib/flowy-db/migrations/.gitkeep | 0 .../2021-07-09-063045_flowy-user/down.sql | 2 + .../2021-07-09-063045_flowy-user/up.sql | 8 ++ rust-lib/flowy-db/src/database.rs | 17 +++ rust-lib/flowy-db/src/errors.rs | 16 +++ rust-lib/flowy-db/src/lib.rs | 15 ++ rust-lib/flowy-db/src/schema.rs | 8 ++ rust-lib/flowy-infra/Cargo.toml | 16 +++ rust-lib/flowy-infra/src/errors.rs | 22 +++ rust-lib/flowy-infra/src/lib.rs | 13 ++ rust-lib/flowy-infra/src/sqlite/database.rs | 36 +++++ rust-lib/flowy-infra/src/sqlite/mod.rs | 5 + rust-lib/flowy-infra/src/sqlite/pool.rs | 134 ++++++++++++++++++ rust-lib/flowy-user/Cargo.toml | 2 + rust-lib/flowy-user/src/domain/database.rs | 26 ++++ rust-lib/flowy-user/src/domain/mod.rs | 1 + rust-lib/flowy-user/src/error.rs | 1 - rust-lib/flowy-user/src/errors.rs | 15 ++ rust-lib/flowy-user/src/lib.rs | 2 +- scripts/install_diesel.sh | 4 + 25 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 rust-lib/flowy-db/.env create mode 100644 rust-lib/flowy-db/Cargo.toml create mode 100644 rust-lib/flowy-db/diesel.toml create mode 100644 rust-lib/flowy-db/migrations/.gitkeep create mode 100644 rust-lib/flowy-db/migrations/2021-07-09-063045_flowy-user/down.sql create mode 100644 rust-lib/flowy-db/migrations/2021-07-09-063045_flowy-user/up.sql create mode 100644 rust-lib/flowy-db/src/database.rs create mode 100644 rust-lib/flowy-db/src/errors.rs create mode 100644 rust-lib/flowy-db/src/lib.rs create mode 100644 rust-lib/flowy-db/src/schema.rs create mode 100644 rust-lib/flowy-infra/Cargo.toml create mode 100644 rust-lib/flowy-infra/src/errors.rs create mode 100644 rust-lib/flowy-infra/src/lib.rs create mode 100644 rust-lib/flowy-infra/src/sqlite/database.rs create mode 100644 rust-lib/flowy-infra/src/sqlite/mod.rs create mode 100644 rust-lib/flowy-infra/src/sqlite/pool.rs create mode 100644 rust-lib/flowy-user/src/domain/database.rs delete mode 100644 rust-lib/flowy-user/src/error.rs create mode 100644 rust-lib/flowy-user/src/errors.rs create mode 100755 scripts/install_diesel.sh diff --git a/.idea/appflowy_client.iml b/.idea/appflowy_client.iml index 2afc7ac296..6d9312f7fb 100644 --- a/.idea/appflowy_client.iml +++ b/.idea/appflowy_client.iml @@ -16,6 +16,8 @@ + + diff --git a/rust-lib/Cargo.toml b/rust-lib/Cargo.toml index 8fb1e8ac2a..482c6924dc 100644 --- a/rust-lib/Cargo.toml +++ b/rust-lib/Cargo.toml @@ -8,6 +8,8 @@ members = [ "flowy-ast", "flowy-derive", "flowy-test", + "flowy-infra", + "flowy-db", ] [profile.dev] diff --git a/rust-lib/flowy-db/.env b/rust-lib/flowy-db/.env new file mode 100644 index 0000000000..2fc14c6e02 --- /dev/null +++ b/rust-lib/flowy-db/.env @@ -0,0 +1 @@ +DATABASE_URL=/tmp/database.sql \ No newline at end of file diff --git a/rust-lib/flowy-db/Cargo.toml b/rust-lib/flowy-db/Cargo.toml new file mode 100644 index 0000000000..0c625e10a8 --- /dev/null +++ b/rust-lib/flowy-db/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "flowy-db" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +diesel = {version = "1.4.7", features = ["sqlite"]} +diesel_derives = {version = "1.4.1", features = ["sqlite"]} +diesel_migrations = {version = "1.4.0", features = ["sqlite"]} +flowy-infra = {path = "../flowy-infra"} \ No newline at end of file diff --git a/rust-lib/flowy-db/diesel.toml b/rust-lib/flowy-db/diesel.toml new file mode 100644 index 0000000000..92267c829f --- /dev/null +++ b/rust-lib/flowy-db/diesel.toml @@ -0,0 +1,5 @@ +# For documentation on how to configure this file, +# see diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema.rs" diff --git a/rust-lib/flowy-db/migrations/.gitkeep b/rust-lib/flowy-db/migrations/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rust-lib/flowy-db/migrations/2021-07-09-063045_flowy-user/down.sql b/rust-lib/flowy-db/migrations/2021-07-09-063045_flowy-user/down.sql new file mode 100644 index 0000000000..f1119aa25a --- /dev/null +++ b/rust-lib/flowy-db/migrations/2021-07-09-063045_flowy-user/down.sql @@ -0,0 +1,2 @@ +-- This file should undo anything in `up.sql` +DROP TABLE user_table; \ No newline at end of file diff --git a/rust-lib/flowy-db/migrations/2021-07-09-063045_flowy-user/up.sql b/rust-lib/flowy-db/migrations/2021-07-09-063045_flowy-user/up.sql new file mode 100644 index 0000000000..2ea0349782 --- /dev/null +++ b/rust-lib/flowy-db/migrations/2021-07-09-063045_flowy-user/up.sql @@ -0,0 +1,8 @@ +-- Your SQL goes here + +CREATE TABLE user_table ( + id TEXT NOT NULL PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + password TEXT NOT NULL DEFAULT '', + email TEXT NOT NULL DEFAULT '' +); diff --git a/rust-lib/flowy-db/src/database.rs b/rust-lib/flowy-db/src/database.rs new file mode 100644 index 0000000000..19baa7f115 --- /dev/null +++ b/rust-lib/flowy-db/src/database.rs @@ -0,0 +1,17 @@ +use crate::errors::FlowyDBError; +use diesel_migrations::*; +use flowy_infra::sqlite::*; +use std::path::Path; + +embed_migrations!("../flowy-db/migrations/"); +pub const DB_NAME: &str = "flowy-database.db"; + +pub fn init(storage_path: &str) -> Result { + if !Path::new(storage_path).exists() { + std::fs::create_dir_all(storage_path)?; + } + + let pool_config = PoolConfig::default(); + let database = DataBase::new(storage_path, DB_NAME, pool_config)?; + Ok(database) +} diff --git a/rust-lib/flowy-db/src/errors.rs b/rust-lib/flowy-db/src/errors.rs new file mode 100644 index 0000000000..f7c23d5fb4 --- /dev/null +++ b/rust-lib/flowy-db/src/errors.rs @@ -0,0 +1,16 @@ +use flowy_infra::Error; +use std::io; + +#[derive(Debug)] +pub enum FlowyDBError { + InitError(String), + IOError(String), +} + +impl std::convert::From for FlowyDBError { + fn from(error: flowy_infra::Error) -> Self { FlowyDBError::InitError(format!("{:?}", error)) } +} + +impl std::convert::From for FlowyDBError { + fn from(error: io::Error) -> Self { FlowyDBError::IOError(format!("{:?}", error)) } +} diff --git a/rust-lib/flowy-db/src/lib.rs b/rust-lib/flowy-db/src/lib.rs new file mode 100644 index 0000000000..29e65f3a95 --- /dev/null +++ b/rust-lib/flowy-db/src/lib.rs @@ -0,0 +1,15 @@ +mod database; +mod errors; +mod schema; + +#[macro_use] +extern crate diesel; +#[macro_use] +extern crate diesel_derives; +#[macro_use] +extern crate diesel_migrations; + +pub use flowy_infra::sqlite::DataBase; + +pub use database::init; +pub use errors::*; diff --git a/rust-lib/flowy-db/src/schema.rs b/rust-lib/flowy-db/src/schema.rs new file mode 100644 index 0000000000..49bd45e814 --- /dev/null +++ b/rust-lib/flowy-db/src/schema.rs @@ -0,0 +1,8 @@ +table! { + user_table (id) { + id -> Text, + name -> Text, + password -> Text, + email -> Text, + } +} diff --git a/rust-lib/flowy-infra/Cargo.toml b/rust-lib/flowy-infra/Cargo.toml new file mode 100644 index 0000000000..19c0e20977 --- /dev/null +++ b/rust-lib/flowy-infra/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "flowy-infra" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +r2d2 = "0.8.9" +diesel = {version = "1.4.7", features = ["sqlite"]} +diesel_derives = {version = "1.4.1", features = ["sqlite"]} +diesel_migrations = {version = "1.4.0", features = ["sqlite"]} +lazy_static = "1.4.0" +scheduled-thread-pool = "0.2.5" +error-chain = "=0.12.0" +log = "0.4.11" \ No newline at end of file diff --git a/rust-lib/flowy-infra/src/errors.rs b/rust-lib/flowy-infra/src/errors.rs new file mode 100644 index 0000000000..37bd372834 --- /dev/null +++ b/rust-lib/flowy-infra/src/errors.rs @@ -0,0 +1,22 @@ +use error_chain::{ + error_chain, + error_chain_processing, + impl_error_chain_kind, + impl_error_chain_processed, + impl_extract_backtrace, +}; + +error_chain! { + errors { + UnknownMigrationExists(v: String) { + display("unknown migration version: '{}'", v), + } + } + foreign_links { + R2D2(::r2d2::Error); + Migrations(::diesel_migrations::RunMigrationsError); + Diesel(::diesel::result::Error); + Connection(::diesel::ConnectionError); + Io(::std::io::Error); + } +} diff --git a/rust-lib/flowy-infra/src/lib.rs b/rust-lib/flowy-infra/src/lib.rs new file mode 100644 index 0000000000..86c7ff9ae4 --- /dev/null +++ b/rust-lib/flowy-infra/src/lib.rs @@ -0,0 +1,13 @@ +#[allow(deprecated, clippy::large_enum_variant)] +mod errors; +pub mod sqlite; + +pub use errors::{Error, ErrorKind, Result}; + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/rust-lib/flowy-infra/src/sqlite/database.rs b/rust-lib/flowy-infra/src/sqlite/database.rs new file mode 100644 index 0000000000..7b31f9cf68 --- /dev/null +++ b/rust-lib/flowy-infra/src/sqlite/database.rs @@ -0,0 +1,36 @@ +use crate::{ + errors::*, + sqlite::pool::{ConnectionManager, ConnectionPool, PoolConfig}, +}; +use r2d2::PooledConnection; + +pub struct DataBase { + uri: String, + pool: ConnectionPool, +} + +impl DataBase { + pub fn new(dir: &str, name: &str, pool_config: PoolConfig) -> Result { + let uri = db_file_uri(dir, name); + let pool = ConnectionPool::new(pool_config, &uri)?; + Ok(Self { uri, pool }) + } + + pub fn get_uri(&self) -> &str { &self.uri } + + pub fn get_conn(&self) -> Result> { + let conn = self.pool.get()?; + Ok(conn) + } +} + +pub fn db_file_uri(dir: &str, name: &str) -> String { + use std::path::MAIN_SEPARATOR; + + let mut uri = dir.to_owned(); + if !uri.ends_with(MAIN_SEPARATOR) { + uri.push(MAIN_SEPARATOR); + } + uri.push_str(name); + uri +} diff --git a/rust-lib/flowy-infra/src/sqlite/mod.rs b/rust-lib/flowy-infra/src/sqlite/mod.rs new file mode 100644 index 0000000000..f016f01d04 --- /dev/null +++ b/rust-lib/flowy-infra/src/sqlite/mod.rs @@ -0,0 +1,5 @@ +mod database; +mod pool; + +pub use database::*; +pub use pool::*; diff --git a/rust-lib/flowy-infra/src/sqlite/pool.rs b/rust-lib/flowy-infra/src/sqlite/pool.rs new file mode 100644 index 0000000000..9452675813 --- /dev/null +++ b/rust-lib/flowy-infra/src/sqlite/pool.rs @@ -0,0 +1,134 @@ +use crate::errors::*; +use diesel::{connection::Connection, SqliteConnection}; +use r2d2::{ManageConnection, Pool}; +use scheduled_thread_pool::ScheduledThreadPool; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering::SeqCst}, + Arc, + }, + time::Duration, +}; + +lazy_static::lazy_static! { + static ref DB_POOL: Arc = Arc::new( + ScheduledThreadPool::with_name("db-pool-{}:", 4) + ); +} + +pub struct ConnectionPool { + pub(crate) inner: Pool, +} + +impl std::ops::Deref for ConnectionPool { + type Target = Pool; + + fn deref(&self) -> &Self::Target { &self.inner } +} + +impl ConnectionPool { + pub fn new(config: PoolConfig, uri: T) -> Result + where + T: Into, + { + let manager = ConnectionManager::new(uri); + let thread_pool = DB_POOL.clone(); + let config = Arc::new(config); + + let pool = r2d2::Pool::builder() + .thread_pool(thread_pool) + .min_idle(Some(config.min_idle)) + .max_size(config.max_size) + .max_lifetime(None) + .connection_timeout(config.connection_timeout) + .idle_timeout(Some(config.idle_timeout)) + .build_unchecked(manager); + Ok(ConnectionPool { inner: pool }) + } +} + +#[derive(Default, Debug, Clone)] +pub struct ConnCounter(Arc); + +impl std::ops::Deref for ConnCounter { + type Target = ConnCounterInner; + + fn deref(&self) -> &Self::Target { &*self.0 } +} + +#[derive(Default, Debug)] +pub struct ConnCounterInner { + max_number: AtomicUsize, + current_number: AtomicUsize, +} + +impl ConnCounterInner { + pub fn get_max_num(&self) -> usize { self.max_number.load(SeqCst) } + + pub fn reset(&self) { + // reset max_number to current_number + let _ = self + .max_number + .fetch_update(SeqCst, SeqCst, |_| Some(self.current_number.load(SeqCst))); + } +} + +pub type OnExecFunc = Box Box + Send + Sync>; + +pub struct PoolConfig { + min_idle: u32, + max_size: u32, + connection_timeout: Duration, + idle_timeout: Duration, +} + +impl Default for PoolConfig { + fn default() -> Self { + Self { + min_idle: 1, + max_size: 10, + connection_timeout: Duration::from_secs(10), + idle_timeout: Duration::from_secs(5 * 60), + } + } +} + +impl PoolConfig { + #[allow(dead_code)] + pub fn min_idle(mut self, min_idle: u32) -> Self { + self.min_idle = min_idle; + self + } + + #[allow(dead_code)] + pub fn max_size(mut self, max_size: u32) -> Self { + self.max_size = max_size; + self + } +} + +pub struct ConnectionManager { + db_uri: String, +} + +impl ManageConnection for ConnectionManager { + type Connection = SqliteConnection; + type Error = crate::Error; + + fn connect(&self) -> Result { + if !std::path::PathBuf::from(&self.db_uri).exists() { + log::error!("db file not exists"); + } + Ok(SqliteConnection::establish(&self.db_uri)?) + } + + fn is_valid(&self, conn: &mut Self::Connection) -> Result<()> { + Ok(conn.execute("SELECT 1").map(|_| ())?) + } + + fn has_broken(&self, _conn: &mut Self::Connection) -> bool { false } +} + +impl ConnectionManager { + pub fn new>(uri: S) -> Self { ConnectionManager { db_uri: uri.into() } } +} diff --git a/rust-lib/flowy-user/Cargo.toml b/rust-lib/flowy-user/Cargo.toml index 075b7d0f96..4c0911f977 100644 --- a/rust-lib/flowy-user/Cargo.toml +++ b/rust-lib/flowy-user/Cargo.toml @@ -10,6 +10,8 @@ derive_more = {version = "0.99", features = ["display"]} flowy-dispatch = { path = "../flowy-dispatch" } flowy-log = { path = "../flowy-log" } flowy-derive = { path = "../flowy-derive" } +flowy-db = { path = "../flowy-db" } + tracing = { version = "0.1", features = ["log"] } bytes = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/rust-lib/flowy-user/src/domain/database.rs b/rust-lib/flowy-user/src/domain/database.rs new file mode 100644 index 0000000000..4a5ec24e19 --- /dev/null +++ b/rust-lib/flowy-user/src/domain/database.rs @@ -0,0 +1,26 @@ +use crate::errors::UserError; +use flowy_db::DataBase; +use lazy_static::lazy_static; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + RwLock, +}; + +lazy_static! { + pub static ref DB: RwLock> = RwLock::new(None); +} + +static DB_INIT: AtomicBool = AtomicBool::new(false); + +pub fn init_user_db(dir: &str) -> Result<(), UserError> { + let database = flowy_db::init(dir)?; + *(DB.write()?) = Some(database); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn init_db_test() { init_user_db(".").unwrap(); } +} diff --git a/rust-lib/flowy-user/src/domain/mod.rs b/rust-lib/flowy-user/src/domain/mod.rs index 6f3bffb8ae..36989ea156 100644 --- a/rust-lib/flowy-user/src/domain/mod.rs +++ b/rust-lib/flowy-user/src/domain/mod.rs @@ -1,3 +1,4 @@ pub use user::*; +mod database; pub mod user; diff --git a/rust-lib/flowy-user/src/error.rs b/rust-lib/flowy-user/src/error.rs deleted file mode 100644 index 8b13789179..0000000000 --- a/rust-lib/flowy-user/src/error.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/rust-lib/flowy-user/src/errors.rs b/rust-lib/flowy-user/src/errors.rs new file mode 100644 index 0000000000..c75abcf501 --- /dev/null +++ b/rust-lib/flowy-user/src/errors.rs @@ -0,0 +1,15 @@ +use std::sync::PoisonError; + +#[derive(Debug)] +pub enum UserError { + DBInitFail(String), + PoisonError(String), +} + +impl std::convert::From for UserError { + fn from(error: flowy_db::FlowyDBError) -> Self { UserError::DBInitFail(format!("{:?}", error)) } +} + +impl std::convert::From> for UserError { + fn from(error: PoisonError) -> Self { UserError::PoisonError(format!("{:?}", error)) } +} diff --git a/rust-lib/flowy-user/src/lib.rs b/rust-lib/flowy-user/src/lib.rs index dd59b9f92b..fa3376178b 100644 --- a/rust-lib/flowy-user/src/lib.rs +++ b/rust-lib/flowy-user/src/lib.rs @@ -1,5 +1,5 @@ mod domain; -mod error; +mod errors; pub mod event; mod handlers; pub mod module; diff --git a/scripts/install_diesel.sh b/scripts/install_diesel.sh new file mode 100755 index 0000000000..d900911f09 --- /dev/null +++ b/scripts/install_diesel.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +brew install sqlite3 +cargo install diesel_cli --no-default-features --features sqlite \ No newline at end of file