results ref is now a maybe, which is more accurate. printing file names actually works as expected

Author: aviaviavi

README.md

@@ -53,6 +53,14 @@ Currently via config you can:
 - Set files to ignore via a list of regular expressions
 - Specify your own flags to scan for other than the built-ins (TODO, FIXME, XXX)
 
+#### Ignoring Files
+
+Ignore as many files as you can! Large autogenerated files will slow Toodles
+down quite a bit. Check the output of the server to see any files/folders that
+may be causing slowness for your repo and add them to the `ignore` section your
+`.toodles.yaml` If the performance of Toodles is not good enough for your use
+case, please open an issue.
+
 ### Scanned Languages
 
 These languages will be scanned for any TODO's:

app/Main.hs

@@ -16,15 +16,16 @@ import           Text.Printf              (printf)
 main :: IO ()
 main = do
   userArgs <- toodlesArgs >>= setAbsolutePath
-  sResults <- runFullSearch userArgs
   case userArgs of
-    (ToodlesArgs _ _ _ _ True _) -> mapM_ (putStrLn . prettyFormat) $ todos sResults
+    (ToodlesArgs _ _ _ _ True _) -> do
+      sResults <- runFullSearch userArgs
+      mapM_ (putStrLn . prettyFormat) $ todos sResults
     _ -> do
-          let webPort = fromMaybe 9001 $ port userArgs
-          ref <- newIORef sResults
-          dataDir <- (++ "/web") <$> getDataDir
-          putStrLn $ "serving on " ++ show webPort
-          run webPort $ app $ ToodlesState ref dataDir
+      let webPort = fromMaybe 9001 $ port userArgs
+      ref <- newIORef Nothing
+      dataDir <- (++ "/web") <$> getDataDir
+      putStrLn $ "serving on " ++ show webPort
+      run webPort $ app $ ToodlesState ref dataDir
 
 prettyFormat :: TodoEntry -> String
 prettyFormat (TodoEntryHead _ l a p n entryPriority f _ _ _ _ _ _) =

src/Parse.hs

@@ -13,7 +13,6 @@ import           Data.Maybe                 (fromJust, fromMaybe, isJust,
 import           Data.Text                  (Text)
 import qualified Data.Text                  as T
 import           Data.Void                  (Void)
-import           Debug.Trace
 import           Text.Megaparsec
 import           Text.Megaparsec.Char
 import qualified Text.Megaparsec.Char.Lexer as L
@@ -212,14 +211,14 @@ nextState a b = error ("No next state for " ++ show a ++ "   " ++ show b)
 runTodoParser :: [UserFlag] -> SourceFile -> [TodoEntry]
 runTodoParser us (SourceFile path ls) =
   let parsedTodoLines =
-        trace ("path done: " ++ path) $ foldl
+        foldl
           (fileParseFoldFn
           us
           path)
           (ParseStateUnknown, [])
           (zip [1 ..] ls)
       groupedTodos = foldl foldTodoHelper ([], False) (snd parsedTodoLines)
-  in trace path fst groupedTodos
+  in fst groupedTodos
 
   where
     -- fold fn to concatenate todos that a multiple, single line comments

src/Server.hs

@@ -1,7 +1,7 @@
+{-# LANGUAGE BangPatterns        #-}
 {-# LANGUAGE DataKinds           #-}
 {-# LANGUAGE OverloadedStrings   #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-
 {-# LANGUAGE TypeOperators       #-}
 
 module Server where
@@ -67,13 +67,16 @@ root (ToodlesState _ dPath) path =
 
 showRawFile :: ToodlesState -> Integer -> Handler Html
 showRawFile (ToodlesState ref _) eId = do
-    (TodoListResult r _) <- liftIO $ readIORef ref
-    let entry = find (\t -> entryId t == eId) r
-    liftIO $
-        maybe
-        (return "Not found")
-        (\e -> addAnchors <$> readFile (sourceFile e))
-        entry
+    storedResults <- liftIO $ readIORef ref
+    case storedResults of
+      (Just (TodoListResult r _)) -> do
+        let entry = find (\t -> entryId t == eId) r
+        liftIO $
+            maybe
+            (return "Not found")
+            (\e -> addAnchors <$> readFile (sourceFile e))
+            entry
+      Nothing -> error "no files to show"
 
     where
     addAnchors :: String -> Html
@@ -87,17 +90,20 @@ showRawFile (ToodlesState ref _) eId = do
 
 editTodos :: ToodlesState -> EditTodoRequest -> Handler Text
 editTodos (ToodlesState ref _) req = do
-  (TodoListResult r _) <- liftIO $ readIORef ref
-  let editedList =
-        map
-          (\t ->
-             if willEditTodo req t
-               then editTodo req t
-               else t)
-          r
-      editedFilteredList = filter (willEditTodo req) editedList
-  _ <- mapM_ recordUpdates editedFilteredList
-  return "{}"
+  storedResults <- liftIO $ readIORef ref
+  case storedResults of
+    (Just (TodoListResult r _)) -> do
+      let editedList =
+            map
+              (\t ->
+                if willEditTodo req t
+                  then editTodo req t
+                  else t)
+              r
+          editedFilteredList = filter (willEditTodo req) editedList
+      _ <- mapM_ recordUpdates editedFilteredList
+      return "{}"
+    Nothing -> error "no stored todos to edit"
   where
     willEditTodo :: EditTodoRequest -> TodoEntry -> Bool
     willEditTodo editRequest entry = entryId entry `elem` editIds editRequest
@@ -189,14 +195,17 @@ updateTodoLinesInFile f todo = do
 
 deleteTodos :: ToodlesState -> DeleteTodoRequest -> Handler Text
 deleteTodos (ToodlesState ref _) req = do
-    refVal@(TodoListResult r _) <- liftIO $ readIORef ref
-    let toDelete = filter (\t -> entryId t `elem` ids req) r
-    liftIO $ doUntilNull removeAndAdjust toDelete
-    let remainingResults = filter (\t -> entryId t `notElem` map entryId toDelete) r
-    let updatedResults = foldl (flip adjustLinesAfterDeletionOf) remainingResults toDelete
-    let remainingResultsRef = refVal { todos = updatedResults }
-    _ <- liftIO $ atomicModifyIORef' ref (const (remainingResultsRef, remainingResultsRef))
-    return "{}"
+    storedResults <- liftIO $ readIORef ref
+    case storedResults of
+      (Just refVal@(TodoListResult r _)) -> do
+        let toDelete = filter (\t -> entryId t `elem` ids req) r
+        liftIO $ doUntilNull removeAndAdjust toDelete
+        let remainingResults = filter (\t -> entryId t `notElem` map entryId toDelete) r
+        let updatedResults = foldl (flip adjustLinesAfterDeletionOf) remainingResults toDelete
+        let remainingResultsRef = refVal { todos = updatedResults }
+        _ <- liftIO $ atomicModifyIORef' ref (const (Just remainingResultsRef, Just remainingResultsRef))
+        return "{}"
+      Nothing -> error "no stored todos"
 
     where
 
@@ -239,14 +248,17 @@ setAbsolutePath args = do
     return $ args {directory = absolute}
 
 getFullSearchResults :: ToodlesState -> Bool -> IO TodoListResult
-getFullSearchResults (ToodlesState ref _) recompute =
-  if recompute
+getFullSearchResults (ToodlesState ref _) recompute = do
+  result <- readIORef ref
+  if recompute || isNothing result
     then do
       putStrLn "refreshing todo's"
       userArgs <- toodlesArgs >>= setAbsolutePath
       sResults <- runFullSearch userArgs
-      atomicModifyIORef' ref (const (sResults, sResults))
-    else putStrLn "cached read" >> readIORef ref
+      atomicModifyIORef' ref (const (Just sResults, sResults))
+    else do
+      putStrLn "cached read"
+      return $ fromMaybe (error "tried to read from the cache when there wasn't anything there") result
 
 runFullSearch :: ToodlesArgs -> IO TodoListResult
 runFullSearch userArgs = do
@@ -259,8 +271,8 @@ runFullSearch userArgs = do
         $ putStrLn $ "[WARNING] Invalid .toodles.yaml: " ++ show config
     let config' = fromRight (ToodlesConfig [] []) config
     allFiles <- getAllFiles config' projectRoot
-    let parsedTodos = concatMap (runTodoParser $ userFlag userArgs ++ flags config') allFiles
-        filteredTodos = filter (filterSearch (assignee_search userArgs)) parsedTodos
+    parsedTodos <- concat <$> mapM (parseFileAndLog userArgs config') allFiles
+    let filteredTodos = filter (filterSearch (assignee_search userArgs)) parsedTodos
         resultList = limitSearch filteredTodos $ limit_results userArgs
         indexedResults = map (\(i, r) -> r {entryId = i}) $ zip [1 ..] resultList
     return $ TodoListResult indexedResults ""
@@ -274,6 +286,14 @@ runFullSearch userArgs = do
     limitSearch todoList 0 = todoList
     limitSearch todoList n = take n todoList
 
+parseFileAndLog :: ToodlesArgs -> ToodlesConfig -> SourceFile -> IO [TodoEntry]
+parseFileAndLog  userArgs config f = do
+  -- the strictness is so we can print "done" when we're actually done
+  !_ <- putStrLn $ fullPath f
+  !result <- return (runTodoParser (userFlag userArgs ++ flags config) f)
+  !_ <- putStrLn "done"
+  return result
+
 getAllFiles :: ToodlesConfig -> FilePath -> IO [SourceFile]
 getAllFiles (ToodlesConfig ignoredPaths _) basePath =
   E.catch

src/Types.hs

@@ -23,7 +23,7 @@ data SourceFile = SourceFile
 type LineNumber = Integer
 
 data ToodlesState = ToodlesState
-  { results  :: IORef TodoListResult,
+  { results  :: IORef (Maybe TodoListResult),
     dataPath :: FilePath
   }
 

web/html/index.html

@@ -121,8 +121,8 @@
             <div class="main-container container content column is-three-quarters">
                 <div class="loading-spinner" v-show="loading"><a class="button is-loading">Loading</a> Loading
                 TODO's. If your project is large, this can take a few moments.
-                Try configuring toodles to ignore large and/or autogenerated
-                files.
+                Try configuring toodles to ignore large, autogenerated, or
+                third-party files.
                 <a target="_blank" href="https://github.com/aviaviavi/toodles/blob/master/.toodles.yaml">More info on configuring toodles</a>
                 </div>
                 <table class="table todo-list is-striped is-hoverable">