Merge pull request #56 from aviaviavi/performance-fixes

Document file ignores for performance
aviaviavi · Nov 28, 2018 · c33cad2 · c33cad2
2 parents a8b73a7 + a3f8a53
commit c33cad2
Show file tree

Hide file tree

Showing 9 changed files with 138 additions and 94 deletions.
diff --git a/.toodles.yaml b/.toodles.yaml
@@ -1,8 +1,12 @@
+# Putting a .toodles.yaml file in the root of your repo will give toodles
+# any project specific configuration you'd like to set.
+
 # `ignore`: Specify a list of regular expressions.
 # Toodles will ignore a file that matches any of the entires
 ignore:
 - test.js
 - stack-work
+- node_modules
 - Spec.hs
 # `flags` specify other keywords you might want to scan other than TODO
 # Hardcoded ones include TODO, FIXME, and XXX

diff --git a/README.md b/README.md
@@ -53,6 +53,14 @@ Currently via config you can:
 - Set files to ignore via a list of regular expressions
 - Specify your own flags to scan for other than the built-ins (TODO, FIXME, XXX)
 
+#### Ignoring Files
+
+Ignore as many files as you can! Large autogenerated files will slow Toodles
+down quite a bit. Check the output of the server to see any files/folders that
+may be causing slowness for your repo and add them to the `ignore` section your
+`.toodles.yaml` If the performance of Toodles is not good enough for your use
+case, please open an issue.
+
 ### Scanned Languages
 
 These languages will be scanned for any TODO's:

diff --git a/app/Main.hs b/app/Main.hs
@@ -16,15 +16,16 @@ import           Text.Printf              (printf)
 main :: IO ()
 main = do
   userArgs <- toodlesArgs >>= setAbsolutePath
-  sResults <- runFullSearch userArgs
   case userArgs of
-    (ToodlesArgs _ _ _ _ True _) -> mapM_ (putStrLn . prettyFormat) $ todos sResults
+    (ToodlesArgs _ _ _ _ True _) -> do
+      sResults <- runFullSearch userArgs
+      mapM_ (putStrLn . prettyFormat) $ todos sResults
     _ -> do
-          let webPort = fromMaybe 9001 $ port userArgs
-          ref <- newIORef sResults
-          dataDir <- (++ "/web") <$> getDataDir
-          putStrLn $ "serving on " ++ show webPort
-          run webPort $ app $ ToodlesState ref dataDir
+      let webPort = fromMaybe 9001 $ port userArgs
+      ref <- newIORef Nothing
+      dataDir <- (++ "/web") <$> getDataDir
+      putStrLn $ "serving on " ++ show webPort
+      run webPort $ app $ ToodlesState ref dataDir
 
 prettyFormat :: TodoEntry -> String
 prettyFormat (TodoEntryHead _ l a p n entryPriority f _ _ _ _ _ _) =

diff --git a/package.yaml b/package.yaml
@@ -1,5 +1,5 @@
 name:                toodles
-version:             1.0.0
+version:             1.0.1
 github:              "aviaviavi/toodles"
 license:             MIT
 author:              "Avi Press"

diff --git a/src/Server.hs b/src/Server.hs
@@ -1,7 +1,7 @@
+{-# LANGUAGE BangPatterns        #-}
 {-# LANGUAGE DataKinds           #-}
 {-# LANGUAGE OverloadedStrings   #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-
 {-# LANGUAGE TypeOperators       #-}
 
 module Server where
@@ -67,13 +67,16 @@ root (ToodlesState _ dPath) path =
 
 showRawFile :: ToodlesState -> Integer -> Handler Html
 showRawFile (ToodlesState ref _) eId = do
-    (TodoListResult r _) <- liftIO $ readIORef ref
-    let entry = find (\t -> entryId t == eId) r
-    liftIO $
-        maybe
-        (return "Not found")
-        (\e -> addAnchors <$> readFile (sourceFile e))
-        entry
+    storedResults <- liftIO $ readIORef ref
+    case storedResults of
+      (Just (TodoListResult r _)) -> do
+        let entry = find (\t -> entryId t == eId) r
+        liftIO $
+            maybe
+            (return "Not found")
+            (\e -> addAnchors <$> readFile (sourceFile e))
+            entry
+      Nothing -> error "no files to show"
 
     where
     addAnchors :: String -> Html
@@ -87,17 +90,20 @@ showRawFile (ToodlesState ref _) eId = do
 
 editTodos :: ToodlesState -> EditTodoRequest -> Handler Text
 editTodos (ToodlesState ref _) req = do
-  (TodoListResult r _) <- liftIO $ readIORef ref
-  let editedList =
-        map
-          (\t ->
-             if willEditTodo req t
-               then editTodo req t
-               else t)
-          r
-      editedFilteredList = filter (willEditTodo req) editedList
-  _ <- mapM_ recordUpdates editedFilteredList
-  return "{}"
+  storedResults <- liftIO $ readIORef ref
+  case storedResults of
+    (Just (TodoListResult r _)) -> do
+      let editedList =
+            map
+              (\t ->
+                if willEditTodo req t
+                  then editTodo req t
+                  else t)
+              r
+          editedFilteredList = filter (willEditTodo req) editedList
+      _ <- mapM_ recordUpdates editedFilteredList
+      return "{}"
+    Nothing -> error "no stored todos to edit"
   where
     willEditTodo :: EditTodoRequest -> TodoEntry -> Bool
     willEditTodo editRequest entry = entryId entry `elem` editIds editRequest
@@ -189,14 +195,17 @@ updateTodoLinesInFile f todo = do
 
 deleteTodos :: ToodlesState -> DeleteTodoRequest -> Handler Text
 deleteTodos (ToodlesState ref _) req = do
-    refVal@(TodoListResult r _) <- liftIO $ readIORef ref
-    let toDelete = filter (\t -> entryId t `elem` ids req) r
-    liftIO $ doUntilNull removeAndAdjust toDelete
-    let remainingResults = filter (\t -> entryId t `notElem` map entryId toDelete) r
-    let updatedResults = foldl (flip adjustLinesAfterDeletionOf) remainingResults toDelete
-    let remainingResultsRef = refVal { todos = updatedResults }
-    _ <- liftIO $ atomicModifyIORef' ref (const (remainingResultsRef, remainingResultsRef))
-    return "{}"
+    storedResults <- liftIO $ readIORef ref
+    case storedResults of
+      (Just refVal@(TodoListResult r _)) -> do
+        let toDelete = filter (\t -> entryId t `elem` ids req) r
+        liftIO $ doUntilNull removeAndAdjust toDelete
+        let remainingResults = filter (\t -> entryId t `notElem` map entryId toDelete) r
+        let updatedResults = foldl (flip adjustLinesAfterDeletionOf) remainingResults toDelete
+        let remainingResultsRef = refVal { todos = updatedResults }
+        _ <- liftIO $ atomicModifyIORef' ref (const (Just remainingResultsRef, Just remainingResultsRef))
+        return "{}"
+      Nothing -> error "no stored todos"
 
     where
 
@@ -239,14 +248,17 @@ setAbsolutePath args = do
     return $ args {directory = absolute}
 
 getFullSearchResults :: ToodlesState -> Bool -> IO TodoListResult
-getFullSearchResults (ToodlesState ref _) recompute =
-  if recompute
+getFullSearchResults (ToodlesState ref _) recompute = do
+  result <- readIORef ref
+  if recompute || isNothing result
     then do
       putStrLn "refreshing todo's"
       userArgs <- toodlesArgs >>= setAbsolutePath
       sResults <- runFullSearch userArgs
-      atomicModifyIORef' ref (const (sResults, sResults))
-    else putStrLn "cached read" >> readIORef ref
+      atomicModifyIORef' ref (const (Just sResults, sResults))
+    else do
+      putStrLn "cached read"
+      return $ fromMaybe (error "tried to read from the cache when there wasn't anything there") result
 
 runFullSearch :: ToodlesArgs -> IO TodoListResult
 runFullSearch userArgs = do
@@ -259,8 +271,8 @@ runFullSearch userArgs = do
         $ putStrLn $ "[WARNING] Invalid .toodles.yaml: " ++ show config
     let config' = fromRight (ToodlesConfig [] []) config
     allFiles <- getAllFiles config' projectRoot
-    let parsedTodos = concatMap (runTodoParser $ userFlag userArgs ++ flags config') allFiles
-        filteredTodos = filter (filterSearch (assignee_search userArgs)) parsedTodos
+    parsedTodos <- concat <$> mapM (parseFileAndLog userArgs config') allFiles
+    let filteredTodos = filter (filterSearch (assignee_search userArgs)) parsedTodos
         resultList = limitSearch filteredTodos $ limit_results userArgs
         indexedResults = map (\(i, r) -> r {entryId = i}) $ zip [1 ..] resultList
     return $ TodoListResult indexedResults ""
@@ -274,6 +286,14 @@ runFullSearch userArgs = do
     limitSearch todoList 0 = todoList
     limitSearch todoList n = take n todoList
 
+parseFileAndLog :: ToodlesArgs -> ToodlesConfig -> SourceFile -> IO [TodoEntry]
+parseFileAndLog  userArgs config f = do
+  -- the strictness is so we can print "done" when we're actually done
+  !_ <- putStrLn $ fullPath f
+  !result <- return (runTodoParser (userFlag userArgs ++ flags config) f)
+  !_ <- putStrLn "done"
+  return result
+
 getAllFiles :: ToodlesConfig -> FilePath -> IO [SourceFile]
 getAllFiles (ToodlesConfig ignoredPaths _) basePath =
   E.catch

diff --git a/src/Types.hs b/src/Types.hs
@@ -23,7 +23,7 @@ data SourceFile = SourceFile
 type LineNumber = Integer
 
 data ToodlesState = ToodlesState
-  { results  :: IORef TodoListResult,
+  { results  :: IORef (Maybe TodoListResult),
     dataPath :: FilePath
   }
 

diff --git a/toodles.cabal b/toodles.cabal
@@ -4,10 +4,10 @@ cabal-version: 1.12
 --
 -- see: https://github.com/sol/hpack
 --
--- hash: f27f81e3ac0847f1de1a670672ef4ebac62f9a14080ada290cee6cfe7d752dea
+-- hash: 95013a4e9f97f67cc56541af010980f8b66d76da59819ff6aef8c68ffdb9bc27
 
 name:           toodles
-version:        1.0.0
+version:        1.0.1
 synopsis:       Manage the TODO entries in your code
 description:    Toodles scrapes your entire repository for TODO entries and organizes them so you can manage your project directly from the code. View, filter, sort, and edit your TODO\'s with an easy to use web application. When you make changes via toodles, the edits will be applied directly the TODO entries in your code. When you\'re done, commit and push your changes to share them with your team!
 category:       Project Management

diff --git a/web/css/toodles.css b/web/css/toodles.css
@@ -67,15 +67,21 @@
 
 .tag-block{
     margin-top: .3em;
+}
+
+.tag-column {
+    max-width: 200px;
+}
+
+.todo-item-customAttributes {
     max-width: 200px;
 }
 
 .tag-item {
-    /* background-color: #E9F1F7; */
     background-color: #4F6D7A;
     line-height: 2;
     color: white;
-    font-size: .75em;
+    font-size: .66em;
     margin-right: .2em;
     padding: .3em;
     text-align: center;
@@ -101,7 +107,7 @@ td, th {
     margin-right: .2em;
     padding: .3em;
     line-height: 2;
-    font-size: .75em;
+    font-size: .66em;
 }
 
 .priority-column {
@@ -133,10 +139,6 @@ td, th {
     background-color: #dbdbdb;
 }
 
-.top-div {
-    overflow-x: scroll;
-}
-
 .toodles-nav-title-text {
     padding-left: 5px;
 }

diff --git a/web/html/index.html b/web/html/index.html
@@ -39,7 +39,7 @@
                     </div>
                     <div class="navbar-item deselect-all" v-on:click="deselectAll" v-show="todos.length && todos.every(t => t.selected)">
                         <span class="icon">
-                           <i class="fa fa-square"></i>
+                            <i class="fa fa-square"></i>
                         </span>
                         <span>Deselect All</span>
                     </div>
@@ -119,48 +119,57 @@
                 </span>
             </div>
             <div class="main-container container content column is-three-quarters">
-                <div class="loading-spinner" v-show="loading"><a class="button is-loading">Loading</a> Loading TODO's. If your project is large, this can take a few moments...</div>
-                <table class="table todo-list is-striped is-hoverable">
-                    <thead>
-                        <tr>
-                            <td></td>
-                            <th v-on:click="sortTodos('priority')" class="sortable priority-column">
-                                Priority
-                                <span class="icon">
-                                    <i class="fa fa-sort"></i>
-                                </span>
-                            </th>
-                            <th>Flag</th>
-                            <th class="todo-item-body">Body</th>
-                            <th>Assignee</th>
-                            <th>Tags</th>
-                        </tr>
-                    </thead>
-                    <tr class="todo-item" v-for="todo in todos" v-if="todo.body.indexOf(todoSearch) !== -1 || (todo.assignee || '').indexOf(todoSearch) !== -1 || (todo.tags || []).toString().indexOf(todoSearch) !== -1 || (todo.flag || '').indexOf(todoSearch) !== -1" @change="updateTodo('top level')" @click="toggleTodo(todo)">
-                        <td><input type="checkbox" v-model="todo.selected"></td>
-                        <td class="todo-item-priority" >
-                            <div v-show="todo.priority !== undefined">{{ todo.priority }}</div>
-                        </td>
-                        <td class="todo-item-flag" >{{ todo.flag }}</td>
-                        <td class='todo-item-body'>
-                            <div class="todo-source-link" @click="stopPropagation"><a :href="'/source_file/' + todo.id + '#line-' + todo.lineNumber" target="_blank">{{ todo.sourceFile }}:{{ todo.lineNumber}}</a></div>
-                            <div>{{ todo.body }}</div>
-                        </td>
-                        <td class="todo-item-assignee" >{{ todo.assignee }}</td>
-                        <td class="todo-item-customAttributes">
-                            <div v-show="Object.keys(todo.customAttributes).length" class="attribute-block">
-                                <span v-for="entry in Object.entries(todo.customAttributes)" class="attribute-item" @change="updateTodo('here')">
-                                    <span >{{entry[0]}}</span>=<span >{{entry[1]}}</span>
-                                </span>
-                            </div>
-                            <div class="tag-block" v-show="Object.keys(todo.tags).length">
-                                <span v-for="tag in todo.tags" class="tag-item">
-                                    <span >{{tag}}</span>
-                                </span>
-                            </div>
-                        </td>
-                    </tr>
-                </table>
+                <div class="loading-spinner" v-show="loading"><a class="button is-loading">Loading</a> Loading
+                    TODO's. If your project is large, this can take a few moments.
+                    Try configuring toodles to ignore large, autogenerated, or
+                    third-party files.
+                    <a target="_blank" href="https://github.com/aviaviavi/toodles/blob/master/.toodles.yaml">More info on configuring toodles</a>
+                </div>
+                <div class="box">
+                    <div class="table-container">
+                        <table class="table is-striped is-narrow is-hoverable is-fullwidth table-container todo-list">
+                            <thead>
+                                <tr>
+                                    <td></td>
+                                    <th v-on:click="sortTodos('priority')" class="sortable priority-column">
+                                        Priority
+                                        <span class="icon">
+                                            <i class="fa fa-sort"></i>
+                                        </span>
+                                    </th>
+                                    <th>Flag</th>
+                                    <th class="todo-item-body">Body</th>
+                                    <th>Assignee</th>
+                                    <th class="todo-item-tags">Tags</th>
+                                </tr>
+                            </thead>
+                            <tr class="todo-item" v-for="todo in todos" v-if="todo.body.indexOf(todoSearch) !== -1 || (todo.assignee || '').indexOf(todoSearch) !== -1 || (todo.tags || []).toString().indexOf(todoSearch) !== -1 || (todo.flag || '').indexOf(todoSearch) !== -1" @change="updateTodo('top level')" @click="toggleTodo(todo)">
+                                <td><input type="checkbox" v-model="todo.selected"></td>
+                                <td class="todo-item-priority" >
+                                    <div v-show="todo.priority !== undefined">{{ todo.priority }}</div>
+                                </td>
+                                <td class="todo-item-flag" >{{ todo.flag }}</td>
+                                <td class='todo-item-body'>
+                                    <div class="todo-source-link" @click="stopPropagation"><a :href="'/source_file/' + todo.id + '#line-' + todo.lineNumber" target="_blank">{{ todo.sourceFile }}:{{ todo.lineNumber}}</a></div>
+                                    <div>{{ todo.body }}</div>
+                                </td>
+                                <td class="todo-item-assignee" >{{ todo.assignee }}</td>
+                                <td class="todo-item-customAttributes">
+                                    <div v-show="Object.keys(todo.customAttributes).length" class="attribute-block">
+                                        <span v-for="entry in Object.entries(todo.customAttributes)" class="attribute-item" @change="updateTodo('here')">
+                                            <span >{{entry[0]}}</span>=<span >{{entry[1]}}</span>
+                                        </span>
+                                    </div>
+                                    <div class="tag-block" v-show="Object.keys(todo.tags).length">
+                                        <span v-for="tag in todo.tags" class="tag-item">
+                                            <span >{{tag}}</span>
+                                        </span>
+                                    </div>
+                                </td>
+                            </tr>
+                        </table>
+                    </div>
+                </div>
             </div>
         </div>
     </div>