sql: set journal_mode to WAL

rfcs/2023-09-28-journal-mode-wal.md

@@ -0,0 +1,30 @@
+# Switching database to WAL mode
+
+## Problem
+
+1. Slow writes when sending messages to large groups.
+
+A possible solution is batching multiple writes into a single transaction, which is attempted for sending messages in #847 / #3067. The problem with that approach is that it substantially complicates the code and has to be done for other scenarios separately (e.g., broadcasting profile updates, which is even more complex as different messages have to be sent to different contacts, to account for preference overrides).
+
+2. Conflicts for the database access from multiple processes (iOS app and NSE).
+
+A possible solution is better coordination of access than currently implemented, but it is substantially more complex, particularly if additional extensions are added.
+
+## Solution
+
+A proposed solution is to increase page_size to 16kb (from 4kb) and switch to WAL mode. This should improve write performance and reduce conflicts.
+
+Problems with this soltion:
+- old versions of the app won't be taking into account WAL file when exporting. Possible solutions are:
+  - make it non-reversible change (that is, without down migration).
+  - checkpoint and switch database to DELETE mode when exporting.
+- windows closes the database connection when the app is stopped, so we can no longer do any operations prior to exporting without providing database key. Possible solutions are:
+  - always checkpoint and move to DELETE mode when stopping and move back to WAL mode when starting.
+  - what else?
+
+Switching to 16kb block also requires a process:
+- set it first
+- run VACUUM (this will change block size)
+- only then the database can be switched to WAL mode
+
+If the database is already in WAL mode it needs to be switched to DELETE mode before block size change will happen on VACUUM.

src/Simplex/Messaging/Agent/Store/SQLite.hs

@@ -32,6 +32,8 @@ module Simplex.Messaging.Agent.Store.SQLite
     connectSQLiteStore,
     closeSQLiteStore,
     openSQLiteStore,
+    checkpointSQLiteStore,
+    setSQLiteModeWAL,
     sqlString,
     execSQL,
     upMigration, -- used in tests
@@ -270,7 +272,7 @@ import Simplex.Messaging.Parsers (blobFieldParser, dropPrefix, fromTextField_, s
 import Simplex.Messaging.Protocol
 import qualified Simplex.Messaging.Protocol as SMP
 import Simplex.Messaging.Transport.Client (TransportHost)
-import Simplex.Messaging.Util (bshow, eitherToMaybe, groupOn, ifM, ($>>=), (<$$>))
+import Simplex.Messaging.Util (bshow, eitherToMaybe, groupOn, ifM, whenM, ($>>=), (<$$>))
 import Simplex.Messaging.Version
 import System.Directory (copyFile, createDirectoryIfMissing, doesFileExist)
 import System.Exit (exitFailure)
@@ -365,7 +367,11 @@ migrateSchema st migrations confirmMigrations = do
     confirm err = confirmOrExit $ migrationErrorDescription err
     run ms = do
       let f = dbFilePath st
+          fWal = f <> "-wal"
+          fShm = f <> "-shm"
       copyFile f (f <> ".bak")
+      whenM (doesFileExist fWal) $ copyFile fWal (f <> ".bak-wal")
+      whenM (doesFileExist fShm) $ copyFile fShm (f <> ".bak-shm")
       Migrations.run st ms
       pure $ Right ()
 
@@ -395,10 +401,11 @@ connectDB path key = do
   pure db
   where
     prepare db = do
-      let exec = SQLite3.exec $ SQL.connectionHandle $ DB.conn db
-      unless (null key) . exec $ "PRAGMA key = " <> sqlString key <> ";"
-      exec . fromQuery $
+      unless (null key) . execSQL_ db $ "PRAGMA key = " <> sqlString key <> ";"
+      execSQL_ db . fromQuery $
         [sql|
+          PRAGMA page_size = 16384;
+          PRAGMA journal_mode = WAL;
           PRAGMA busy_timeout = 100;
           PRAGMA foreign_keys = ON;
           -- PRAGMA trusted_schema = OFF;
@@ -409,8 +416,9 @@ connectDB path key = do
 closeSQLiteStore :: SQLiteStore -> IO ()
 closeSQLiteStore st@SQLiteStore {dbClosed} =
   ifM (readTVarIO dbClosed) (putStrLn "closeSQLiteStore: already closed") $
-    withConnection st $ \conn -> do
-      DB.close conn
+    withConnection st $ \db -> do
+      execSQL_ db "PRAGMA wal_checkpoint(TRUNCATE);"
+      DB.close db
       atomically $ writeTVar dbClosed True
 
 openSQLiteStore :: SQLiteStore -> String -> IO ()
@@ -427,6 +435,18 @@ openSQLiteStore SQLiteStore {dbConnection, dbFilePath, dbClosed} key =
             putTMVar dbConnection DB.Connection {conn, slow}
             writeTVar dbClosed False
 
+checkpointSQLiteStore :: SQLiteStore -> IO ()
+checkpointSQLiteStore st =
+  withConnection st (`execSQL_` "PRAGMA wal_checkpoint(TRUNCATE);")
+
+setSQLiteModeWAL :: SQLiteStore -> Bool -> IO ()
+setSQLiteModeWAL st walMode =
+  withConnection st (`execSQL_` q)
+  where
+    q
+      | walMode = "PRAGMA journal_mode = WAL;"
+      | otherwise = "PRAGMA wal_checkpoint(TRUNCATE); PRAGMA journal_mode = DELETE;"
+
 sqlString :: String -> Text
 sqlString s = quote <> T.replace quote "''" (T.pack s) <> quote
   where
@@ -444,6 +464,9 @@ sqlString s = quote <> T.replace quote "''" (T.pack s) <> quote
 --   auto_vacuum <- DB.query_ db "PRAGMA auto_vacuum;" :: IO [[Int]]
 --   print $ path <> " auto_vacuum: " <> show auto_vacuum
 
+execSQL_ :: DB.Connection -> Text -> IO ()
+execSQL_ = SQLite3.exec . SQL.connectionHandle . DB.conn
+
 execSQL :: DB.Connection -> Text -> IO [Text]
 execSQL db query = do
   rs <- newIORef []