Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-18 22:16:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Squashed commit of the following:

Browse Source 
commit 6bc5d1b90e
Author: holger krekel <holger@merlinux.eu>
Date:   Sun Jul 21 22:56:37 2019 +0200

    fix fmt

commit 197d94ad9d
Merge: 7ce337c 686678c
Author: holger krekel <holger@merlinux.eu>
Date:   Sun Jul 21 22:51:16 2019 +0200

    Merge remote-tracking branch 'origin/master' into eventlogging

commit 7ce337c6d0
Author: holger krekel <holger@merlinux.eu>
Date:   Sun Jul 21 22:44:27 2019 +0200

    left-over error logging

commit 10148d2e43
Author: holger krekel <holger@merlinux.eu>
Date:   Sun Jul 21 22:03:17 2019 +0200

    ignore non-utf8 parts of header fields (add comment why it shouldn't happen)
    don't throw error if no sql rows are returned

commit 69dc237ee3
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Sun Jul 21 12:56:04 2019 +0200

    fix(receive_imf): remove recursive sql call

commit df5464ea80
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Sat Jul 20 17:05:24 2019 +0200

    fix: blocked is an optional value

commit e4bf9956a5
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Sat Jul 20 16:50:56 2019 +0200

    fix(msg): handle optional in_reply_to

commit d353d9d9d8
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Sat Jul 20 16:17:25 2019 +0200

    fix(chat): remove recursive sql usage

commit 1ad45ed4d6
Author: holger krekel <holger@merlinux.eu>
Date:   Sat Jul 20 15:14:11 2019 +0200

    fix rust fmt

commit 496e980a17
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Sat Jul 20 14:34:20 2019 +0200

    use forked rusqlite

commit fa09e46ed9
Author: holger krekel <holger@merlinux.eu>
Date:   Sat Jul 20 12:37:51 2019 +0200

    another pace where we might (and in my case did) get invalid utf8

commit d6de420b9a
Author: holger krekel <holger@merlinux.eu>
Date:   Sat Jul 20 12:30:48 2019 +0200

    fix some string issues, introduce to_string_lossy such that to_string() continues to panic on non-utf8

commit 38eb708db8
Author: holger krekel <holger@merlinux.eu>
Date:   Sat Jul 20 01:17:53 2019 +0200

    for now make to_string() less strict as we often don't want to crash the whole app just because some non-proper utf8 came in (through a message we can't neccesarily congtrol)

commit 7a59da5f8f
Author: holger krekel <holger@merlinux.eu>
Date:   Fri Jul 19 22:48:39 2019 +0200

    fix linting

commit f13a1d4a2f
Author: holger krekel <holger@merlinux.eu>
Date:   Fri Jul 19 22:46:58 2019 +0200

    fix some test flakyness

commit 7b3a450918
Author: holger krekel <holger@merlinux.eu>
Date:   Fri Jul 19 22:35:07 2019 +0200

    - fix saved_mime test which broke to improper conversion of
      imf_raw_not_terminated
    - some cargo.toml updates no clue where they come from
    - log Message-ID for received messages

commit 169923b102
Author: holger krekel <holger@merlinux.eu>
Date:   Fri Jul 19 12:31:22 2019 +0200

    formatting

commit 42688a0622
Author: holger krekel <holger@merlinux.eu>
Date:   Fri Jul 19 12:24:56 2019 +0200

    remove some print statements

commit 35f3c0edd1
Merge: e7a2362 f58b1d6
Author: holger krekel <holger@merlinux.eu>
Date:   Fri Jul 19 10:25:21 2019 +0200

    Merge branch 'master' into eventlogging

commit e7a236264a
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Thu Jul 18 23:20:20 2019 +0200

    print invalid strings

commit aaa5b820d9
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Thu Jul 18 23:12:35 2019 +0200

    cleanup

commit e7f0745010
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Thu Jul 18 23:03:57 2019 +0200

    reduce direc usage of CString

commit c68e7ae14e
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Thu Jul 18 22:47:47 2019 +0200

    audit use of to_cstring and fix ub

commit 618087e5a7
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Thu Jul 18 21:38:52 2019 +0200

    fix(imap): body ptr lifetime

commit 245abb8384
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Thu Jul 18 19:44:10 2019 +0200

    remove debug

commit a3e1042001
Author: dignifiedquire <dignifiedquire@users.noreply.github.com>
Date:   Thu Jul 18 18:30:54 2019 +0200

    fix some things, add more debugging statements

commit 7b7ce9348f
Author: holger krekel <holger@merlinux.eu>
Date:   Thu Jul 18 15:11:57 2019 +0200

    fix python lint issues

commit 7a4808ba0d
Author: holger krekel <holger@merlinux.eu>
Date:   Thu Jul 18 14:35:54 2019 +0200

    cargofmt

commit 8f240f7153
Author: holger krekel <holger@merlinux.eu>
Date:   Thu Jul 18 14:03:57 2019 +0200

    (dig,hpk) pull out job collection from sql query/lock logic

commit 7d0b5d8abb
Author: holger krekel <holger@merlinux.eu>
Date:   Thu Jul 18 12:52:02 2019 +0200

    remove print statements and fix a crash

commit ee317cb1b5
Author: holger krekel <holger@merlinux.eu>
Date:   Thu Jul 18 11:38:10 2019 +0200

    fix some merge issues

commit 7b736fe635
Author: holger krekel <holger@merlinux.eu>
Date:   Thu Jul 18 11:16:38 2019 +0200

    (dig,hpk) add test and fix for wrong dbs

commit c7db15352a
Merge: 0b37167 0c5015d
Author: holger krekel <holger@merlinux.eu>
Date:   Thu Jul 18 09:59:44 2019 +0200

    Merge branch 'master' into eventlogging

commit 0b37167be8
Author: holger krekel <holger@merlinux.eu>
Date:   Thu Jul 18 00:06:05 2019 +0200

    address @dignifiedquire comments

commit 5cac4b5076
Author: holger krekel <holger@merlinux.eu>
Date:   Wed Jul 17 12:47:22 2019 +0200

    remove spurious print

commit 475a41beb3
Author: holger krekel <holger@merlinux.eu>
Date:   Wed Jul 17 12:31:12 2019 +0200

    address @dignifiedquire rustyness comment and fix changelog

commit ad4be80b4e
Author: holger krekel <holger@merlinux.eu>
Date:   Wed Jul 17 10:25:25 2019 +0200

    make smtp/imap connect() return bool instead of c-int

commit 8737c1d142
Author: holger krekel <holger@merlinux.eu>
Date:   Wed Jul 17 09:26:33 2019 +0200

    cleanup some parts, add comments

commit 964fe466cc
Author: holger krekel <holger@merlinux.eu>
Date:   Tue Jul 16 20:05:41 2019 +0200

    wip-commit which passes all tests with proper finalization

commit 43936e7db7
Author: holger krekel <holger@merlinux.eu>
Date:   Tue Jul 16 16:17:42 2019 +0200

    snapshot of my current debugging state

commit 0e80ce9c39
Author: holger krekel <holger@merlinux.eu>
Date:   Tue Jul 16 12:57:19 2019 +0200

    more aggressively skip perform API when threads are closing

commit c652bae68a
Author: holger krekel <holger@merlinux.eu>
Date:   Tue Jul 16 12:06:05 2019 +0200

    intermediate wip commit

commit bc904a495d
Author: holger krekel <holger@merlinux.eu>
Date:   Tue Jul 16 11:18:56 2019 +0200

    add some logging, and a more precise teardown for online python tests

commit 8d99444c6a
Author: holger krekel <holger@merlinux.eu>
Date:   Tue Jul 16 00:22:12 2019 +0200

    fix std

commit 9dab53e0af
Author: holger krekel <holger@merlinux.eu>
Date:   Tue Jul 16 00:20:54 2019 +0200

    rustfmt

commit 360089ac74
Author: holger krekel <holger@merlinux.eu>
Date:   Tue Jul 16 00:03:49 2019 +0200

    remove some debugging

commit e892c5cf4d
Author: holger krekel <holger@merlinux.eu>
Date:   Mon Jul 15 23:31:30 2019 +0200

    fix test for events

commit 9ad4c9a6fe
Author: holger krekel <holger@merlinux.eu>
Date:   Mon Jul 15 22:51:57 2019 +0200

    wip try test that we see INFO events from the core
This commit is contained in:
holger krekel
2019-07-21 23:31:14 +02:00
parent 686678c96c
commit 8089559958
45 changed files with 1165 additions and 836 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
src/sql.rs
View File

@@ -1,7 +1,8 @@
use std::collections::HashSet;
use std::sync::RwLock;
use std::sync::{Arc, RwLock};
use rusqlite::{Connection, OpenFlags, Statement, NO_PARAMS};
use thread_local_object::ThreadLocal;
use crate::constants::*;
use crate::context::Context;
@@ -16,12 +17,14 @@ const DC_OPEN_READONLY: usize = 0x01;
/// A wrapper around the underlying Sqlite3 object.
pub struct Sql {
pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
in_use: Arc<ThreadLocal<String>>,
}
impl Sql {
pub fn new() -> Sql {
Sql {
pool: RwLock::new(None),
in_use: Arc::new(ThreadLocal::new()),
}
}
@@ -31,6 +34,7 @@ impl Sql {
pub fn close(&self, context: &Context) {
let _ = self.pool.write().unwrap().take();
self.in_use.remove();
// drop closes the connection
info!(context, 0, "Database closed.");
@@ -53,6 +57,7 @@ impl Sql {
P: IntoIterator,
P::Item: rusqlite::ToSql,
{
self.start_stmt(sql.to_string());
self.with_conn(|conn| conn.execute(sql, params).map_err(Into::into))
}
@@ -60,22 +65,25 @@ impl Sql {
where
G: FnOnce(&Connection) -> Result<T>,
{
match &*self.pool.read().unwrap() {
let res = match &*self.pool.read().unwrap() {
Some(pool) => {
let conn = pool.get()?;
g(&conn)
}
None => Err(Error::SqlNoConnection),
}
};
self.in_use.remove();
res
}
pub fn prepare<G, H>(&self, sql: &str, g: G) -> Result<H>
where
G: FnOnce(Statement<'_>) -> Result<H>,
G: FnOnce(Statement<'_>, &Connection) -> Result<H>,
{
self.start_stmt(sql.to_string());
self.with_conn(|conn| {
let stmt = conn.prepare(sql)?;
let res = g(stmt)?;
let res = g(stmt, conn)?;
Ok(res)
})
}
@@ -84,6 +92,7 @@ impl Sql {
where
G: FnOnce(Statement<'_>, Statement<'_>, &Connection) -> Result<H>,
{
self.start_stmt(format!("{} - {}", sql1, sql2));
self.with_conn(|conn| {
let stmt1 = conn.prepare(sql1)?;
let stmt2 = conn.prepare(sql2)?;
@@ -109,8 +118,8 @@ impl Sql {
F: FnMut(&rusqlite::Row) -> rusqlite::Result<T>,
G: FnMut(rusqlite::MappedRows<F>) -> Result<H>,
{
self.start_stmt(sql.as_ref().to_string());
self.with_conn(|conn| {
eprintln!("query_map {}", sql.as_ref());
let mut stmt = conn.prepare(sql.as_ref())?;
let res = stmt.query_map(params, f)?;
g(res)
@@ -124,6 +133,7 @@ impl Sql {
P: IntoIterator,
P::Item: rusqlite::ToSql,
{
self.start_stmt(sql.to_string());
self.with_conn(|conn| {
let mut stmt = conn.prepare(sql)?;
let res = stmt.exists(params)?;
@@ -137,11 +147,12 @@ impl Sql {
P::Item: rusqlite::ToSql,
F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
{
self.start_stmt(sql.as_ref().to_string());
self.with_conn(|conn| conn.query_row(sql.as_ref(), params, f).map_err(Into::into))
}
pub fn table_exists(&self, name: impl AsRef<str>) -> bool {
self.with_conn(|conn| Ok(table_exists(conn, name)))
self.with_conn(|conn| table_exists(conn, name))
.unwrap_or_default()
}
@@ -259,18 +270,27 @@ impl Sql {
pub fn get_config_int64(&self, context: &Context, key: impl AsRef<str>) -> Option<i64> {
self.get_config(context, key).and_then(|r| r.parse().ok())
}
fn start_stmt(&self, stmt: impl AsRef<str>) {
if let Some(query) = self.in_use.get_cloned() {
let bt = backtrace::Backtrace::new();
eprintln!("old query: {}", query);
eprintln!("Connection is already used from this thread: {:?}", bt);
panic!("Connection is already used from this thread");
}
self.in_use.set(stmt.as_ref().to_string());
}
}
fn table_exists(conn: &Connection, name: impl AsRef<str>) -> bool {
fn table_exists(conn: &Connection, name: impl AsRef<str>) -> Result<bool> {
let mut exists = false;
conn.pragma(None, "table_info", &format!("{}", name.as_ref()), |_row| {
// will only be executed if the info was found
exists = true;
Ok(())
})
.expect("bad sqlite state");
exists
})?;
Ok(exists)
}
fn open(
@@ -841,6 +861,18 @@ pub fn get_rowid(
table: impl AsRef<str>,
field: impl AsRef<str>,
value: impl AsRef<str>,
) -> u32 {
sql.start_stmt("get rowid".to_string());
sql.with_conn(|conn| Ok(get_rowid_with_conn(context, conn, table, field, value)))
.unwrap_or_else(|_| 0)
}
pub fn get_rowid_with_conn(
context: &Context,
conn: &Connection,
table: impl AsRef<str>,
field: impl AsRef<str>,
value: impl AsRef<str>,
) -> u32 {
// alternative to sqlite3_last_insert_rowid() which MUST NOT be used due to race conditions, see comment above.
// the ORDER BY ensures, this function always returns the most recent id,
@@ -852,7 +884,7 @@ pub fn get_rowid(
value.as_ref()
);
match sql.query_row(&query, NO_PARAMS, |row| row.get::<_, u32>(0)) {
match conn.query_row(&query, NO_PARAMS, |row| row.get::<_, u32>(0)) {
Ok(id) => id,
Err(err) => {
error!(
@@ -863,7 +895,6 @@ pub fn get_rowid(
}
}
}
pub fn get_rowid2(
context: &Context,
sql: &Sql,
@@ -873,6 +904,7 @@ pub fn get_rowid2(
field2: impl AsRef<str>,
value2: i32,
) -> u32 {
sql.start_stmt("get rowid2".to_string());
sql.with_conn(|conn| {
Ok(get_rowid2_with_conn(
context, conn, table, field, value, field2, value2,
@@ -972,31 +1004,36 @@ pub fn housekeeping(context: &Context) {
}
let entry = entry.unwrap();
let name_f = entry.file_name();
let name_c = to_cstring(name_f.to_string_lossy());
let name_c = unsafe { to_cstring(name_f.to_string_lossy()) };
if unsafe {
is_file_in_use(&mut files_in_use, 0 as *const libc::c_char, name_c.as_ptr())
} || unsafe {
is_file_in_use(
&mut files_in_use,
b".increation\x00" as *const u8 as *const libc::c_char,
name_c.as_ptr(),
)
} || unsafe {
is_file_in_use(
&mut files_in_use,
b".waveform\x00" as *const u8 as *const libc::c_char,
name_c.as_ptr(),
)
} || unsafe {
is_file_in_use(
&mut files_in_use,
b"-preview.jpg\x00" as *const u8 as *const libc::c_char,
name_c.as_ptr(),
)
} {
if unsafe { is_file_in_use(&mut files_in_use, 0 as *const libc::c_char, name_c) }
|| unsafe {
is_file_in_use(
&mut files_in_use,
b".increation\x00" as *const u8 as *const libc::c_char,
name_c,
)
}
|| unsafe {
is_file_in_use(
&mut files_in_use,
b".waveform\x00" as *const u8 as *const libc::c_char,
name_c,
)
}
|| unsafe {
is_file_in_use(
&mut files_in_use,
b"-preview.jpg\x00" as *const u8 as *const libc::c_char,
name_c,
)
}
{
unsafe { free(name_c as *mut _) };
continue;
}
unsafe { free(name_c as *mut _) };
unreferenced_count += 1;
match std::fs::metadata(entry.path()) {
@@ -1028,8 +1065,11 @@ pub fn housekeeping(context: &Context) {
unreferenced_count,
entry.file_name()
);
let path = to_cstring(entry.path().to_str().unwrap());
unsafe { dc_delete_file(context, path.as_ptr()) };
unsafe {
let path = to_cstring(entry.path().to_str().unwrap());
dc_delete_file(context, path);
free(path as *mut _);
}
}
}
Err(err) => {
@@ -1087,14 +1127,16 @@ fn maybe_add_from_param(
context
.sql
.query_row(query, NO_PARAMS, |row| {
let v = to_cstring(row.get::<_, String>(0)?);
unsafe {
dc_param_set_packed(param, v.as_ptr() as *const libc::c_char);
let file = dc_param_get(param, param_id, 0 as *const libc::c_char);
let v = to_cstring(row.get::<_, String>(0)?);
dc_param_set_packed(param, v as *const _);
let file = dc_param_get(param, param_id, 0 as *const _);
if !file.is_null() {
maybe_add_file(files_in_use, as_str(file));
free(file as *mut libc::c_void);
}
free(v as *mut _);
}
Ok(())
})
@@ -1128,7 +1170,6 @@ mod test {
maybe_add_file(&mut files, "$BLOBDIR/world.txt");
maybe_add_file(&mut files, "world2.txt");
println!("{:?}", files);
assert!(unsafe {
is_file_in_use(
&mut files,