Only check stderr and stdout if files explicitly exists

Also adds a test case where we check both modes:

  * Assert the overall output (ignoring file descriptors)
  * Assert output of stdout and stderr respectively

Currently the test may fail because the `.out` file will have an
arbitrary interleaving of the output from stderr and stdout.

cabal-testsuite/PackageTests/FileDescriptors/M.hs

@@ -0,0 +1 @@
+module M where

cabal-testsuite/PackageTests/FileDescriptors/Setup.hs

@@ -0,0 +1,2 @@
+main :: IO ()
+main = fail "Setup called despite `build-type:Simple`"

cabal-testsuite/PackageTests/FileDescriptors/cabal.out

@@ -0,0 +1,8 @@
+# cabal build
+Resolving dependencies...
+Build profile: -w ghc-<GHCVER> -O1
+In order, the following will be built:
+ - my-0 (lib) (first run)
+Configuring library for my-0..
+Preprocessing library for my-0..
+Building library for my-0..

cabal-testsuite/PackageTests/FileDescriptors/cabal.project

@@ -0,0 +1 @@
+packages: .

cabal-testsuite/PackageTests/FileDescriptors/cabal.stderr

@@ -0,0 +1,7 @@
+Resolving dependencies...
+Build profile: -w ghc-<GHCVER> -O1
+In order, the following will be built:
+ - my-0 (lib) (first run)
+Configuring library for my-0..
+Preprocessing library for my-0..
+Building library for my-0..

cabal-testsuite/PackageTests/FileDescriptors/cabal.stdout

@@ -0,0 +1 @@
+# cabal build

cabal-testsuite/PackageTests/FileDescriptors/cabal.test.hs

@@ -0,0 +1,4 @@
+import Test.Cabal.Prelude
+
+main :: IO ()
+main = cabalTest $ cabal "build" ["all"]

cabal-testsuite/PackageTests/FileDescriptors/my.cabal

@@ -0,0 +1,9 @@
+cabal-version:       3.0
+name:                my
+version:             0
+-- tests that output goes to the correct file descriptors
+
+library
+  exposed-modules:     M
+  build-depends:       base
+  default-language:    Haskell2010

cabal-testsuite/src/Test/Cabal/DecodeShowBuildInfo.hs

@@ -36,7 +36,8 @@ recordBuildInfo pkgName cname = do
     recordLog $ Result
       { resultExitCode = ExitSuccess
       , resultCommand = "build --enable-build-info"
-      , result = buildInfo
+      -- TODO: Consider if these three fields are instantiated correctly.
+      , resultOut = buildInfo
       , resultStdout = buildInfo
       , resultStderr = mempty
       }

cabal-testsuite/src/Test/Cabal/Monad.hs

@@ -35,6 +35,7 @@ module Test.Cabal.Monad (
     testSourceCopyDir,
     testCabalDir,
     testUserCabalConfigFile,
+    testActualFileOut,
     testActualFileStdout,
     testActualFileStderr,
     -- * Skipping tests
@@ -353,25 +354,36 @@ runTestM mode m = withSystemTempDirectory "cabal-testsuite" $ \tmp_dir -> do
     check_expect accept = do
         env <- getTestEnv
         norm_env <- mkNormalizerEnv
-        actualStdout <- normalizeOutput norm_env <$> liftIO (readFileOrEmpty (testActualFileStdout env))
-        expectStdout <- liftIO $ readFileOrEmpty (testExpectFileStdout env)
-        actualStderr <- normalizeOutput norm_env <$> liftIO (readFileOrEmpty (testActualFileStderr env))
-        expectStderr <- liftIO $ readFileOrEmpty (testExpectFileStderr env)
-        let testDiff actual expect = when (words actual /= words expect) $ do
+        let testDiff actual actualNormalized expect expectNormalized = when (words actual /= words expect) $ do
               -- First try whitespace insensitive diff
-              let actual_fp = testNormalizedActualFileStdout env
-                  expect_fp = testNormalizedExpectFileStdout env
-              liftIO $ writeFile actual_fp actual
-              liftIO $ writeFile expect_fp expect
+              -- let actualNormalized = testNormalizedActualFileStdout env
+                  -- expectNormalized = testNormalizedExpectFileStdout env
+              liftIO $ writeFile actualNormalized actual
+              liftIO $ writeFile expectNormalized expect
               liftIO $ putStrLn "Actual output differs from expected:"
-              b <- diff ["-uw"] expect_fp actual_fp
-              unless b . void $ diff ["-u"] expect_fp actual_fp
+              b <- diff ["-uw"] expectNormalized actualNormalized
+              unless b . void $ diff ["-u"] expectNormalized actualNormalized
               if accept
                   then do liftIO $ putStrLn "Accepting new output."
                           liftIO $ writeFileNoCR (testExpectFileStdout env) actual
                   else liftIO $ exitWith (ExitFailure 1)
-        testDiff actualStdout expectStdout
-        testDiff actualStderr expectStderr
+        -- actualOut    <- normalizeOutput norm_env <$> liftIO (readFileOrEmpty (testActualFileOut env))
+        -- expectOut    <- liftIO $ readFileOrEmpty (testExpectFileOut env)
+        -- actualStdout <- normalizeOutput norm_env <$> liftIO (readFileOrEmpty (testActualFileStdout env))
+        -- expectStdout <- liftIO $ readFileOrEmpty (testExpectFileStdout env)
+        -- actualStderr <- normalizeOutput norm_env <$> liftIO (readFileOrEmpty (testActualFileStderr env))
+        -- expectStderr <- liftIO $ readFileOrEmpty (testExpectFileStderr env)
+        forM_
+          [ (testActualFileOut    env, testNormalizedActualFileOut env   , testExpectFileOut env   , testNormalizedExpectFileOut env)
+          , (testActualFileStdout env, testNormalizedActualFileStdout env, testExpectFileStdout env, testNormalizedExpectFileStdout env)
+          , (testActualFileStderr env, testNormalizedActualFileStderr env, testExpectFileStderr env, testNormalizedExpectFileStderr env)
+          ] $ \(actualFile, actualFileNormalized, expectFile, expectFileNormalized) -> do
+            exists <- liftIO $ doesFileExist expectFile
+            liftIO $ print (expectFile, exists)
+            when exists $ do
+              actualOut <- normalizeOutput norm_env <$> liftIO (readFileOrEmpty actualFile)
+              expectOut <- liftIO $ readFileOrEmpty expectFile
+              testDiff actualOut actualFileNormalized expectOut expectFileNormalized
 
 readFileOrEmpty :: FilePath -> IO String
 readFileOrEmpty f = readFile f `E.catch` \e ->
@@ -600,6 +612,30 @@ testCabalDir env = testHomeDir env </> ".cabal"
 testUserCabalConfigFile :: TestEnv -> FilePath
 testUserCabalConfigFile env = testCabalDir env </> "config"
 
+-- data Expected = Expected | Actual
+-- data Normalized = Normalized | NotNormalized
+-- data FileDescriptor = Out | Stdout | Stderr
+
+-- testFile :: Expected -> Normalized -> FileDescriptor -> TestEnv -> FilePath
+-- testFile e n f env = sourceDir </> testName env <.> suffix
+--   where
+--   suffix
+--     =   case e of { Expected   -> "comp"       ; Actual        -> "" }
+--     <.> case f of { Out        -> "out"        ; Stdout        -> "stdout" ; Stderr -> "stderr" }
+--     <.> case n of { Normalized -> "normalized" ; NotNormalized -> "" }
+--   sourceDir = case (e, n) of
+--     (_, Normalized) -> testWorkDir env
+--     (Actual, _)     -> testWorkDir env
+--     _               -> testSourceDir env
+
+-- | The file where the expected output of the test lives
+testExpectFileOut :: TestEnv -> FilePath
+testExpectFileOut env = testSourceDir env </> testName env <.> "out"
+
+-- | Where we store the actual output
+testActualFileOut :: TestEnv -> FilePath
+testActualFileOut env = testWorkDir env </> testName env <.> "comp.out"
+
 -- | The file where the expected output of the test lives
 testExpectFileStdout :: TestEnv -> FilePath
 testExpectFileStdout env = testSourceDir env </> testName env <.> "stdout"
@@ -616,10 +652,26 @@ testExpectFileStderr env = testSourceDir env </> testName env <.> "stderr"
 testActualFileStderr :: TestEnv -> FilePath
 testActualFileStderr env = testWorkDir env </> testName env <.> "comp.stderr"
 
+-- | Where we will write the normalized actual file (for diffing)
+testNormalizedActualFileOut :: TestEnv -> FilePath
+testNormalizedActualFileOut env = testActualFileOut env <.> "normalized"
+
+-- | Where we will write the normalized expected file (for diffing)
+testNormalizedExpectFileOut :: TestEnv -> FilePath
+testNormalizedExpectFileOut env = testWorkDir env </> testName env <.> "out.normalized"
+
 -- | Where we will write the normalized actual file (for diffing)
 testNormalizedActualFileStdout :: TestEnv -> FilePath
 testNormalizedActualFileStdout env = testActualFileStdout env <.> "normalized"
 
 -- | Where we will write the normalized expected file (for diffing)
 testNormalizedExpectFileStdout :: TestEnv -> FilePath
 testNormalizedExpectFileStdout env = testWorkDir env </> testName env <.> "stdout.normalized"
+
+-- | Where we will write the normalized actual file (for diffing)
+testNormalizedActualFileStderr :: TestEnv -> FilePath
+testNormalizedActualFileStderr env = testActualFileStderr env <.> "normalized"
+
+-- | Where we will write the normalized expected file (for diffing)
+testNormalizedExpectFileStderr :: TestEnv -> FilePath
+testNormalizedExpectFileStderr env = testWorkDir env </> testName env <.> "stderr.normalized"

cabal-testsuite/src/Test/Cabal/Prelude.hs

@@ -598,15 +598,16 @@ recordHeader args = do
             initWorkDir
             liftIO $ putStr str_header
             liftIO $ C.appendFile (testWorkDir env </> "test.log") header
-            liftIO $ C.appendFile (testActualFileStdout env) header
+            liftIO $ C.appendFile (testActualFileOut env) header
 
 recordLog :: Result -> TestM ()
-recordLog Result{resultStdout, resultStderr, resultCommand} = do
+recordLog Result{resultOut, resultStdout, resultStderr, resultCommand} = do
     env <- getTestEnv
     let mode = testRecordMode env
     initWorkDir
     liftIO $ C.appendFile (testWorkDir env </> "test.log")
                          (C.pack $ "+ " ++ resultCommand ++ "\n"
+                            ++ "OUT\n" <> resultOut ++ "\n"
                             ++ "STDOUT\n" <> resultStdout ++ "\n"
                             ++ "STDERR\n" <> resultStderr ++ "\n"
                             ++ "\n")
@@ -618,6 +619,7 @@ recordLog Result{resultStdout, resultStderr, resultCommand} = do
             RecordAll    -> unlines (lines txt)
             RecordMarked -> getMarkedOutput txt
             DoNotRecord  -> ""
+    report (testActualFileOut env) resultOut
     report (testActualFileStdout env) resultStdout
     report (testActualFileStderr env) resultStderr
 

cabal-testsuite/src/Test/Cabal/Run.hs

@@ -21,8 +21,11 @@ import Data.Foldable (traverse_)
 data Result = Result
     { resultExitCode    :: ExitCode
     , resultCommand     :: String
-    , result            :: String
+    -- | Output sent to any file descriptor.
+    , resultOut         :: String
+    -- | Output sent to stdout.
     , resultStdout      :: String
+    -- | Output sent to stderr.
     , resultStderr      :: String
     } deriving Show
 
@@ -92,7 +95,7 @@ run _verbosity mb_cwd env_overrides path0 args input = do
     return Result {
             resultExitCode = exitcode,
             resultCommand = showCommandForUser path args,
-            result = rAll,
+            resultOut = rAll,
             resultStdout = rStdout,
             resultStderr = rStderr
         }