Skip to content

Commit 8355f0c

Browse files
committed
Add separate cache for getPkgConfigDb
Querying pkg-config for the version of every module can be a very expensive operation on some systems. This change adds a separate, per-project, cache for PkgConfigDB; reducing the cost from "every plan change" to "every pkg-config-db change per project". The cache key is composed by the pkg-config configured program and the list of directories reported by pkg-config's pc_path variable. A notice is presented to the user when refreshing the PkgConfigDB.
1 parent b34184e commit 8355f0c

File tree

153 files changed

+268
-51
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

153 files changed

+268
-51
lines changed

cabal-install/src/Distribution/Client/ProjectConfig.hs

+20
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ module Distribution.Client.ProjectConfig
5050
, resolveSolverSettings
5151
, BuildTimeSettings (..)
5252
, resolveBuildTimeSettings
53+
, resolveProgramDb
5354

5455
-- * Checking configuration
5556
, checkBadPerPackageCompilerPaths
@@ -154,6 +155,13 @@ import Distribution.Simple.InstallDirs
154155
)
155156
import Distribution.Simple.Program
156157
( ConfiguredProgram (..)
158+
, ProgramSearchPathEntry (..)
159+
)
160+
import Distribution.Simple.Program.Db
161+
( ProgramDb
162+
, defaultProgramDb
163+
, modifyProgramSearchPath
164+
, userSpecifyPaths
157165
)
158166
import Distribution.Simple.Setup
159167
( Flag (Flag)
@@ -496,6 +504,18 @@ resolveBuildTimeSettings
496504
| isParallelBuild buildSettingNumJobs = False
497505
| otherwise = False
498506

507+
-- | ProgramDb with directly user specified paths
508+
resolveProgramDb :: ProjectConfig -> ProgramDb
509+
resolveProgramDb ProjectConfig{projectConfigLocalPackages = PackageConfig{..}} =
510+
userSpecifyPaths (Map.toList (getMapLast packageConfigProgramPaths))
511+
. modifyProgramSearchPath
512+
( [ ProgramSearchPathDir dir
513+
| dir <- fromNubList packageConfigProgramPathExtra
514+
]
515+
++
516+
)
517+
$ defaultProgramDb
518+
499519
---------------------------------------------
500520
-- Reading and writing project config files
501521
--

cabal-install/src/Distribution/Client/ProjectPlanning.hs

+31-48
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,7 @@ configureCompiler
483483
DistDirLayout
484484
{ distProjectCacheFile
485485
}
486-
ProjectConfig
486+
projectConfig@ProjectConfig
487487
{ projectConfigShared =
488488
ProjectConfigShared
489489
{ projectConfigHcFlavor
@@ -533,15 +533,7 @@ configureCompiler
533533
hcFlavor = flagToMaybe projectConfigHcFlavor
534534
hcPath = flagToMaybe projectConfigHcPath
535535
hcPkg = flagToMaybe projectConfigHcPkg
536-
progdb =
537-
userSpecifyPaths (Map.toList (getMapLast packageConfigProgramPaths))
538-
. modifyProgramSearchPath
539-
( [ ProgramSearchPathDir dir
540-
| dir <- fromNubList packageConfigProgramPathExtra
541-
]
542-
++
543-
)
544-
$ defaultProgramDb
536+
progdb = resolveProgramDb projectConfig
545537

546538
------------------------------------------------------------------------------
547539

@@ -590,6 +582,9 @@ rebuildInstallPlan
590582
progsearchpath <- liftIO $ getSystemSearchPath
591583
let projectConfigMonitored = projectConfig{projectConfigBuildOnly = mempty}
592584

585+
let progdb = resolveProgramDb projectConfig
586+
pkgConfigDB <- getPkgConfigDb verbosity distDirLayout progdb
587+
593588
-- The overall improved plan is cached
594589
rerunIfChanged
595590
verbosity
@@ -609,16 +604,16 @@ rebuildInstallPlan
609604
)
610605
$ do
611606
compilerEtc <- phaseConfigureCompiler projectConfig
612-
_ <- phaseConfigurePrograms projectConfig compilerEtc
613-
(solverPlan, pkgConfigDB, totalIndexState, activeRepos) <-
607+
608+
(solverPlan, totalIndexState, activeRepos) <-
614609
phaseRunSolver
615610
projectConfig
616611
compilerEtc
612+
pkgConfigDB
617613
localPackages
618614
(fromMaybe mempty mbInstalledPackages)
619-
( elaboratedPlan
620-
, elaboratedShared
621-
) <-
615+
616+
(elaboratedPlan, elaboratedShared) <-
622617
phaseElaboratePlan
623618
projectConfig
624619
compilerEtc
@@ -652,55 +647,37 @@ rebuildInstallPlan
652647
phaseConfigureCompiler
653648
:: ProjectConfig
654649
-> Rebuild (Compiler, Platform, ProgramDb)
655-
phaseConfigureCompiler = configureCompiler verbosity distDirLayout
650+
phaseConfigureCompiler projectConfig = do
651+
(compiler, platform, progdb) <- configureCompiler verbosity distDirLayout projectConfig
656652

657-
-- Configuring other programs.
658-
--
659-
-- Having configred the compiler, now we configure all the remaining
660-
-- programs. This is to check we can find them, and to monitor them for
661-
-- changes.
662-
--
663-
-- TODO: [required eventually] we don't actually do this yet.
664-
--
665-
-- We rely on the fact that the previous phase added the program config for
666-
-- all local packages, but that all the programs configured so far are the
667-
-- compiler program or related util programs.
668-
--
669-
phaseConfigurePrograms
670-
:: ProjectConfig
671-
-> (Compiler, Platform, ProgramDb)
672-
-> Rebuild ()
673-
phaseConfigurePrograms projectConfig (_, _, compilerprogdb) = do
674653
-- Users are allowed to specify program locations independently for
675654
-- each package (e.g. to use a particular version of a pre-processor
676655
-- for some packages). However they cannot do this for the compiler
677656
-- itself as that's just not going to work. So we check for this.
678657
liftIO $
679658
checkBadPerPackageCompilerPaths
680-
(configuredPrograms compilerprogdb)
659+
(configuredPrograms progdb)
681660
(getMapMappend (projectConfigSpecificPackage projectConfig))
682661

683-
-- TODO: [required eventually] find/configure other programs that the
684-
-- user specifies.
685-
686-
-- TODO: [required eventually] find/configure all build-tools
687-
-- but note that some of them may be built as part of the plan.
662+
return (compiler, platform, progdb)
688663

689664
-- Run the solver to get the initial install plan.
690665
-- This is expensive so we cache it independently.
691666
--
692667
phaseRunSolver
693668
:: ProjectConfig
694669
-> (Compiler, Platform, ProgramDb)
670+
-> PkgConfigDb
695671
-> [PackageSpecifier UnresolvedSourcePackage]
696672
-> InstalledPackageIndex
697-
-> Rebuild (SolverInstallPlan, PkgConfigDb, IndexUtils.TotalIndexState, IndexUtils.ActiveRepos)
673+
-> Rebuild (SolverInstallPlan, IndexUtils.TotalIndexState, IndexUtils.ActiveRepos)
698674
phaseRunSolver
699675
projectConfig@ProjectConfig
700676
{ projectConfigShared
701677
, projectConfigBuildOnly
702678
}
703679
(compiler, platform, progdb)
680+
pkgConfigDB
704681
localPackages
705682
installedPackages =
706683
rerunIfChanged
@@ -721,13 +698,13 @@ rebuildInstallPlan
721698
progdb
722699
platform
723700
corePackageDbs
701+
724702
(sourcePkgDb, tis, ar) <-
725703
getSourcePackages
726704
verbosity
727705
withRepoCtx
728706
(solverSettingIndexState solverSettings)
729707
(solverSettingActiveRepos solverSettings)
730-
pkgConfigDB <- getPkgConfigDb verbosity progdb
731708

732709
-- TODO: [code cleanup] it'd be better if the Compiler contained the
733710
-- ConfiguredPrograms that it needs, rather than relying on the progdb
@@ -752,7 +729,7 @@ rebuildInstallPlan
752729
Left msg -> do
753730
reportPlanningFailure projectConfig compiler platform localPackages
754731
dieWithException verbosity $ PhaseRunSolverErr msg
755-
Right plan -> return (plan, pkgConfigDB, tis, ar)
732+
Right plan -> return (plan, tis, ar)
756733
where
757734
corePackageDbs :: [PackageDB]
758735
corePackageDbs =
@@ -1038,13 +1015,19 @@ getSourcePackages verbosity withRepoCtx idxState activeRepos = do
10381015
$ repos
10391016
return sourcePkgDbWithTIS
10401017

1041-
getPkgConfigDb :: Verbosity -> ProgramDb -> Rebuild PkgConfigDb
1042-
getPkgConfigDb verbosity progdb = do
1018+
getPkgConfigDb :: Verbosity -> DistDirLayout -> ProgramDb -> Rebuild PkgConfigDb
1019+
getPkgConfigDb verbosity distDirLayout progdb = do
1020+
systemsearchpath <- liftIO $ getSystemSearchPath
1021+
let progsearchpath = getProgramSearchPath progdb
10431022
dirs <- liftIO $ getPkgConfigDbDirs verbosity progdb
1044-
-- Just monitor the dirs so we'll notice new .pc files.
1045-
-- Alternatively we could monitor all the .pc files too.
1046-
traverse_ monitorDirectoryStatus dirs
1047-
liftIO $ readPkgConfigDb verbosity progdb
1023+
rerunIfChanged verbosity fileMonitorPkgConfigDb (systemsearchpath, progsearchpath, dirs) $ do
1024+
liftIO $ notice verbosity "Querying pkg-config database..."
1025+
-- Just monitor the dirs so we'll notice new .pc files.
1026+
-- Alternatively we could monitor all the .pc files too.
1027+
traverse_ monitorDirectoryStatus dirs
1028+
liftIO $ readPkgConfigDb verbosity progdb
1029+
where
1030+
fileMonitorPkgConfigDb = newFileMonitor $ distProjectCacheFile distDirLayout "pkg-config-db"
10481031

10491032
-- | Select the config values to monitor for changes package source hashes.
10501033
packageLocationsSignature

cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external-target.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal-target.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.out

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:
@@ -18,8 +19,7 @@ Configuring library 'indef' instantiated with
1819
Data.Map = containers-<VERSION>:Data.Map
1920
for Includes3-0.1.0.0...
2021
Preprocessing library 'indef' for Includes3-0.1.0.0...
21-
Building library 'indef' instantiated with
22-
Data.Map = containers-<VERSION>:Data.Map
22+
Building library 'indef' instantiated with Data.Map = containers-<VERSION>:Data.Map
2323
for Includes3-0.1.0.0...
2424
Configuring executable 'exe' for Includes3-0.1.0.0...
2525
Preprocessing executable 'exe' for Includes3-0.1.0.0...

cabal-testsuite/PackageTests/Backpack/Includes3/cabal-repo.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

cabal-testsuite/PackageTests/Backpack/Reexport2/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Error:
45
Problem with module re-exports:

cabal-testsuite/PackageTests/Backpack/T6385/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

cabal-testsuite/PackageTests/Backpack/bkpcabal01/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/BuildDeps/DepCycle/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Error:
45
Dependency cycle between the following components:

cabal-testsuite/PackageTests/BuildDeps/InternalLibrary1/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/BuildTargets/UseLocalPackage/use-local-version-of-package.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

cabal-testsuite/PackageTests/BuildTargets/UseLocalPackageForSetup/use-local-package-as-setup-dep.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Error: [Cabal-7107]
67
Could not resolve dependencies:

cabal-testsuite/PackageTests/BuildToolDepends/setup.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/BuildTools/External/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/BuildTools/Internal/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/CmmSources/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-run
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/CmmSourcesDyn/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-run
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/CmmSourcesExe/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-run
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/ConditionalAndImport/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-run
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

cabal-testsuite/PackageTests/ConfigFile/T8487/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O2
45
In order, the following will be built:

cabal-testsuite/PackageTests/Configure/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/CopyHie/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

cabal-testsuite/PackageTests/CustomSegfault/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/CustomWithoutCabal/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

cabal-testsuite/PackageTests/CustomWithoutCabalDefaultMain/cabal.out

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

0 commit comments

Comments
 (0)