diff --git a/.paket/Paket.Restore.targets b/.paket/Paket.Restore.targets index fecbd7e3f8..f84efcaab6 100644 --- a/.paket/Paket.Restore.targets +++ b/.paket/Paket.Restore.targets @@ -5,6 +5,9 @@ true $(MSBuildThisFileDirectory) + $(MSBuildThisFileDirectory)..\ + $(PaketRootPath)paket-files\paket.restore.cached + $(PaketRootPath)paket.lock /Library/Frameworks/Mono.framework/Commands/mono mono @@ -16,29 +19,74 @@ $(PaketToolsPath)paket.bootstrapper.exe "$(PaketBootStrapperExePath)" $(MonoPath) --runtime=v4.0.30319 "$(PaketBootStrapperExePath)" - + true - true + true - - - + + + true + + + + $([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)')) + $([System.IO.File]::ReadAllText('$(PaketLockFilePath)')) + true + false + true + + + + + + - $(MSBuildProjectDirectory)/obj/$(MSBuildProjectFile).references + $(MSBuildProjectDirectory)\obj\$(MSBuildProjectFile).paket.references.cached + + $(MSBuildProjectFullPath).paket.references + + $(MSBuildProjectDirectory)\$(MSBuildProjectName).paket.references + + $(MSBuildProjectDirectory)\paket.references + $(MSBuildProjectDirectory)\obj\$(MSBuildProjectFile).$(TargetFramework).references + true + references-file-or-cache-not-found - + + + $([System.IO.File]::ReadAllText('$(PaketReferencesCachedFilePath)')) + $([System.IO.File]::ReadAllText('$(PaketOriginalReferencesFilePath)')) + references-file + false + + + + + true + target-framework + + + + + + + + + + + - + $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[0]) $([System.String]::Copy('%(PaketReferencesFileLines.Identity)').Split(',')[1]) @@ -80,7 +128,7 @@ - $(MSBuildProjectDirectory)/obj/$(MSBuildProjectFile).references + $(MSBuildProjectDirectory)/$(MSBuildProjectFile) true false true @@ -90,7 +138,7 @@ <_NuspecFiles Include="$(BaseIntermediateOutputPath)*.nuspec"/> - + diff --git a/.paket/paket.targets b/.paket/paket.targets index 2fb5f4d52b..4621967c2e 100644 --- a/.paket/paket.targets +++ b/.paket/paket.targets @@ -6,6 +6,8 @@ true $(MSBuildThisFileDirectory) $(MSBuildThisFileDirectory)..\ + $(PaketRootPath)paket.lock + $(PaketRootPath)paket-files\paket.restore.cached /Library/Frameworks/Mono.framework/Commands/mono mono @@ -48,11 +50,23 @@ RestorePackages; $(BuildDependsOn); + + true + + + + $([System.IO.File]::ReadAllText('$(PaketRestoreCacheFile)')) + $([System.IO.File]::ReadAllText('$(PaketLockFilePath)')) + true + false + true + + diff --git a/Paket.sln b/Paket.sln index 2b7f9e5f73..bf02d55e30 100644 --- a/Paket.sln +++ b/Paket.sln @@ -110,6 +110,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".paket", ".paket", "{62D18A EndProjectSection EndProject Global + GlobalSection(Performance) = preSolution + HasPerformanceSessions = true + EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 01437f2a76..8368a91585 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,3 +1,6 @@ +#### 5.92.0 - 26.08.2017 +* PERFORMANCE: Make restore faster - https://github.com/fsprojects/Paket/pull/2675 + #### 5.91.0 - 26.08.2017 * BUGFIX: fix a bug in the runtime parser - https://github.com/fsprojects/Paket/pull/2665 * BUGFIX: Add props to correct Paket.Restore.targets - https://github.com/fsprojects/Paket/pull/2665 diff --git a/build.fsx b/build.fsx index d5ed0e73fc..1259b40ecf 100644 --- a/build.fsx +++ b/build.fsx @@ -168,6 +168,9 @@ Target "Build" (fun _ -> "SourceLinkCreate" , "true" ] "Rebuild" |> ignore + + // DogFood newly build paket.exe for the dotnet build + setEnvironVar "PaketExePath" (Path.GetFullPath (buildDir @@ "paket.exe")) ) let assertExitCodeZero x = @@ -183,11 +186,14 @@ let runCmdIn workDir exe = let dotnet workDir = runCmdIn workDir "dotnet" Target "DotnetRestoreTools" (fun _ -> + // DogFood newly build paket.exe for the dotnet restore for the tools. + setEnvironVar "PaketExePath" (Path.GetFullPath (buildDir @@ "paket.exe")) DotNetCli.Restore (fun c -> { c with Project = currentDirectory "tools" "tools.fsproj" - ToolPath = dotnetExePath + ToolPath = dotnetExePath }) + setEnvironVar "PaketExePath" null ) Target "DotnetRestore" (fun _ -> @@ -230,6 +236,8 @@ Target "DotnetPackage" (fun _ -> // Run the unit tests using test runner Target "RunTests" (fun _ -> + // Stop bootstrapping in from here (tests should use whatever they want to test). + setEnvironVar "PaketExePath" null !! testAssemblies |> NUnit3 (fun p -> { p with diff --git a/src/Paket.Core/Common/Constants.fs b/src/Paket.Core/Common/Constants.fs index 013a9e8297..b80240eeac 100644 --- a/src/Paket.Core/Common/Constants.fs +++ b/src/Paket.Core/Common/Constants.fs @@ -14,6 +14,8 @@ let [] GithubReleaseDownloadUrl = "https://github.com/fsprojects/Paket let [] LockFileName = "paket.lock" /// 'paket.local' let [] LocalFileName = "paket.local" +/// 'paket.restore.sha512' +let [] RestoreHashFile = "paket.restore.cached" /// 'paket.dependencies' let [] DependenciesFileName = "paket.dependencies" /// '.paket' diff --git a/src/Paket.Core/Common/Utils.fs b/src/Paket.Core/Common/Utils.fs index 1ab75db7e6..0173763844 100644 --- a/src/Paket.Core/Common/Utils.fs +++ b/src/Paket.Core/Common/Utils.fs @@ -1071,6 +1071,14 @@ let removeComment (text:string) = | p1, p2 -> stripComment (min p1 p2) remove 0 +let getSha512Stream (stream:Stream) = + use hasher = System.Security.Cryptography.SHA512.Create() :> System.Security.Cryptography.HashAlgorithm + Convert.ToBase64String(hasher.ComputeHash(stream)) + +let getSha512File (filePath:string) = + use stream = File.OpenRead(filePath) + getSha512Stream stream + // adapted from MiniRx // http://minirx.codeplex.com/ [] diff --git a/src/Paket.Core/Dependencies/NuGetCache.fs b/src/Paket.Core/Dependencies/NuGetCache.fs index 255f72ace1..33dc6299e0 100644 --- a/src/Paket.Core/Dependencies/NuGetCache.fs +++ b/src/Paket.Core/Dependencies/NuGetCache.fs @@ -316,10 +316,7 @@ let rec ExtractPackageToUserFolder(fileName:string, packageName:PackageName, ver let cachedHashFile = Path.Combine(Constants.NuGetCacheFolder,fi.Name + ".sha512") if not <| File.Exists cachedHashFile then - use stream = File.OpenRead(fileName) - let packageSize = stream.Length - use hasher = System.Security.Cryptography.SHA512.Create() :> System.Security.Cryptography.HashAlgorithm - let packageHash = Convert.ToBase64String(hasher.ComputeHash(stream)) + let packageHash = getSha512File fileName File.WriteAllText(cachedHashFile,packageHash) File.Copy(cachedHashFile,targetPackageFileName + ".sha512") diff --git a/src/Paket.Core/Dependencies/PackageResolver.fs b/src/Paket.Core/Dependencies/PackageResolver.fs index d39cdc1836..63c5add36a 100644 --- a/src/Paket.Core/Dependencies/PackageResolver.fs +++ b/src/Paket.Core/Dependencies/PackageResolver.fs @@ -1271,7 +1271,7 @@ let Resolve (getVersionsRaw, getPreferredVersionsRaw, getPackageDetailsRaw, grou // Flag to ensure that we don't hide underlying exceptions in the finally block. let mutable exceptionThrown = false try -#if DEBUG +#if DEBUG && !DOTNETCORE let mutable results = None let mutable error = None // Increase stack size, because we have no tail-call-elimination diff --git a/src/Paket.Core/Installation/RestoreProcess.fs b/src/Paket.Core/Installation/RestoreProcess.fs index f7b06f6bda..ecc5d661c6 100644 --- a/src/Paket.Core/Installation/RestoreProcess.fs +++ b/src/Paket.Core/Installation/RestoreProcess.fs @@ -260,71 +260,90 @@ let createPaketCLIToolsFile (cliTools:ResolvedPackage seq) (fileInfo:FileInfo) = if verbose then tracefn " - %s already up-to-date" fileInfo.FullName -let createProjectReferencesFiles (dependenciesFile:DependenciesFile) (lockFile:LockFile) (projectFile:FileInfo) (referencesFile:ReferencesFile) (resolved:Lazy>) targetFilter (groups:Map) = +let createProjectReferencesFiles (dependenciesFile:DependenciesFile) (lockFile:LockFile) (projectFile:ProjectFile) (referencesFile:ReferencesFile) (resolved:Lazy>) (groups:Map) = + let projectFileInfo = FileInfo projectFile.FileName let list = System.Collections.Generic.List<_>() - let cliTools = System.Collections.Generic.List<_>() - for kv in groups do - let hull,cliToolsInGroup = lockFile.GetOrderedPackageHull(kv.Key,referencesFile) - cliTools.AddRange cliToolsInGroup - - let depsGroup = - match dependenciesFile.Groups |> Map.tryFind kv.Key with - | Some group -> group - | None -> failwithf "Dependencies file '%s' does not contain group '%O' but it is used in '%s'" dependenciesFile.FileName kv.Key lockFile.FileName - - let allDirectPackages = - match referencesFile.Groups |> Map.tryFind kv.Key with - | Some g -> g.NugetPackages |> List.map (fun p -> p.Name) |> Set.ofList - | None -> Set.empty - - - for (key,_,_) in hull do - let restore = - match targetFilter with - | None -> true - | Some targets -> + let hulls = + groups + |> Seq.map (fun kv -> + let hull,cliToolsInGroup = lockFile.GetOrderedPackageHull(kv.Key,referencesFile) + kv.Key, (hull, cliToolsInGroup)) + |> dict + let targets = + ProjectFile.getTargetFramework projectFile + |> Option.toList + |> List.append (ProjectFile.getTargetFrameworks projectFile |> Option.toList |> List.collect (fun item -> String.split [|';'|] item |> List.ofArray)) + |> List.map (fun s -> s, (PlatformMatching.forceExtractPlatforms s |> fun p -> p.ToTargetProfile true)) + |> List.choose (fun (s, c) -> c |> Option.map (fun d -> s, d)) + + // delete stale entries (otherwise we might not recognize stale data on a change later) + let objDir = DirectoryInfo(Path.Combine(projectFileInfo.Directory.FullName,"obj")) + objDir.GetFiles(sprintf "%s*.references" projectFileInfo.Name) + |> Seq.iter (fun f -> f.Delete()) + + // fable 1.0 compat + let oldReferencesFile = FileInfo(Path.Combine(projectFileInfo.Directory.FullName,"obj",projectFileInfo.Name + ".references")) + if oldReferencesFile.Exists then oldReferencesFile.Delete() + + for originalTargetProfileString, targetProfile in targets do + for kv in groups do + let hull,_ = hulls.[kv.Key] + let allDirectPackages = + match referencesFile.Groups |> Map.tryFind kv.Key with + | Some g -> g.NugetPackages |> List.map (fun p -> p.Name) |> Set.ofList + | None -> Set.empty + + for (key,_,_) in hull do + let restore = let resolvedPackage = resolved.Force().[key] match resolvedPackage.Settings.FrameworkRestrictions with | Requirements.ExplicitRestriction restrictions -> - targets - |> Array.exists (fun target -> Requirements.isTargetMatchingRestrictions(restrictions, SinglePlatform target)) + Requirements.isTargetMatchingRestrictions(restrictions, targetProfile) | _ -> true - - if restore then - let _,packageName = key - let direct = allDirectPackages.Contains packageName - let package = resolved.Force().[key] - let line = - packageName.ToString() + "," + - package.Version.ToString() + "," + - (if direct then "Direct" else "Transitive") + "," + - kv.Key.ToString() - - list.Add line - - let output = String.Join(Environment.NewLine,list) - let newFileName = FileInfo(Path.Combine(projectFile.Directory.FullName,"obj",projectFile.Name + ".references")) - if not newFileName.Directory.Exists then - newFileName.Directory.Create() - if output = "" then - if File.Exists(newFileName.FullName) then - File.Delete(newFileName.FullName) - - elif not newFileName.Exists || File.ReadAllText(newFileName.FullName) <> output then - File.WriteAllText(newFileName.FullName,output) - tracefn " - %s created" newFileName.FullName - else - if verbose then - tracefn " - %s already up-to-date" newFileName.FullName + if restore then + let _,packageName = key + let direct = allDirectPackages.Contains packageName + let package = resolved.Force().[key] + let line = + packageName.ToString() + "," + + package.Version.ToString() + "," + + (if direct then "Direct" else "Transitive") + "," + + kv.Key.ToString() + + list.Add line + + let output = String.Join(Environment.NewLine,list) + let newFileName = FileInfo(Path.Combine(projectFileInfo.Directory.FullName,"obj",projectFileInfo.Name + "." + originalTargetProfileString + ".references")) + if not newFileName.Directory.Exists then + newFileName.Directory.Create() + + elif not newFileName.Exists || File.ReadAllText(newFileName.FullName) <> output then + if targetProfile = SinglePlatform (FrameworkIdentifier.DotNetStandard DotNetStandardVersion.V1_6) then + // fable compat + File.WriteAllText(oldReferencesFile.FullName,output) + File.WriteAllText(newFileName.FullName,output) + tracefn " - %s created" newFileName.FullName + else + if verbose then + tracefn " - %s already up-to-date" newFileName.FullName - let paketCLIToolsFileName = FileInfo(Path.Combine(projectFile.Directory.FullName,"obj",projectFile.Name + ".paket.clitools")) + let cliTools = System.Collections.Generic.List<_>() + for kv in groups do + let _,cliToolsInGroup = hulls.[kv.Key] // lockFile.GetOrderedPackageHull(kv.Key,referencesFile) + cliTools.AddRange cliToolsInGroup + + let paketCLIToolsFileName = FileInfo(Path.Combine(projectFileInfo.Directory.FullName,"obj",projectFileInfo.Name + ".paket.clitools")) createPaketCLIToolsFile cliTools paketCLIToolsFileName - - let paketPropsFileName = FileInfo(Path.Combine(projectFile.Directory.FullName,"obj",projectFile.Name + ".paket.props")) + + let paketPropsFileName = FileInfo(Path.Combine(projectFileInfo.Directory.FullName,"obj",projectFileInfo.Name + ".paket.props")) createPaketPropsFile cliTools paketPropsFileName + // Write "cached" file, this way msbuild can check if the references file has changed. + let paketCachedReferencesFileName = FileInfo(Path.Combine(projectFileInfo.Directory.FullName,"obj",projectFileInfo.Name + ".paket.references.cached")) + File.Copy(referencesFile.FileName, paketCachedReferencesFileName.FullName, true) + let CreateScriptsForGroups dependenciesFile lockFile (groups:Map) = let groupsToGenerate = groups @@ -340,8 +359,8 @@ let CreateScriptsForGroups dependenciesFile lockFile (groups:Map Seq.iter (fun sd -> sd.Save dir) -let FindOrCreateReferencesFile projectFileName = - let projectFile = ProjectFile.LoadFromFile projectFileName +let FindOrCreateReferencesFile (projectFile:ProjectFile) = + match projectFile.FindReferencesFile() with | Some fileName -> try @@ -354,7 +373,15 @@ let FindOrCreateReferencesFile projectFileName = Path.Combine(fi.Directory.FullName,Constants.ReferencesFile) ReferencesFile.New fileName - + +let RestoreNewSdkProject dependenciesFile lockFile resolved groups (projectFile:ProjectFile) = + let referencesFile = FindOrCreateReferencesFile projectFile + let projectFileInfo = FileInfo projectFile.FileName + + createAlternativeNuGetConfig projectFileInfo + createProjectReferencesFiles dependenciesFile lockFile projectFile referencesFile resolved groups + referencesFile + let Restore(dependenciesFileName,projectFile,force,group,referencesFileNames,ignoreChecks,failOnChecks,targetFrameworks: string option) = let lockFileName = DependenciesFile.FindLockfile dependenciesFileName let localFileName = DependenciesFile.FindLocalfile dependenciesFileName @@ -362,105 +389,130 @@ let Restore(dependenciesFileName,projectFile,force,group,referencesFileNames,ign let alternativeProjectRoot = None if not lockFileName.Exists then failwithf "%s doesn't exist." lockFileName.FullName - let dependenciesFile = DependenciesFile.ReadFromFile(dependenciesFileName) - - let targetFilter = - targetFrameworks - |> Option.map (fun s -> s.Split(';') |> Array.map FrameworkDetection.Extract |> Array.choose id) - let lockFile,localFile,hasLocalFile = - let lockFile = LockFile.LoadFrom(lockFileName.FullName) - if not localFileName.Exists then - lockFile,LocalFile.empty,false - else - let localFile = - LocalFile.readFile localFileName.FullName - |> returnOrFail - LocalFile.overrideLockFile localFile lockFile,localFile,true - - if not hasLocalFile && not ignoreChecks then - let hasAnyChanges,nugetChanges,remoteFilechanges,hasChanges = DependencyChangeDetection.GetChanges(dependenciesFile,lockFile,false) - - let checkResponse = if failOnChecks then failwithf else traceWarnfn - if hasAnyChanges then - checkResponse "paket.dependencies and paket.lock are out of sync in %s.%sPlease run 'paket install' or 'paket update' to recompute the paket.lock file." lockFileName.Directory.FullName Environment.NewLine - for (group, package, changes) in nugetChanges do - traceWarnfn "Changes were detected for %s/%s" (group.ToString()) (package.ToString()) - for change in changes do - traceWarnfn " - %A" change - - let groups = - match group with - | None -> lockFile.Groups - | Some groupName -> - match lockFile.Groups |> Map.tryFind groupName with - | None -> failwithf "The group %O was not found in the paket.lock file." groupName - | Some group -> [groupName,group] |> Map.ofList - - let resolved = lazy (lockFile.GetGroupedResolution()) - - let referencesFileNames = - match projectFile with - | Some projectFileName -> - let referencesFile = FindOrCreateReferencesFile projectFileName - let projectFileInfo = FileInfo projectFileName - - createAlternativeNuGetConfig projectFileInfo - createProjectReferencesFiles dependenciesFile lockFile projectFileInfo referencesFile resolved targetFilter groups - - [referencesFile.FileName] - | None -> referencesFileNames - - let tasks = - groups - |> Seq.map (fun kv -> - let allPackages = - if List.isEmpty referencesFileNames then - kv.Value.Resolution - |> Seq.map (fun kv -> kv.Key) - else - referencesFileNames - |> List.toSeq - |> computePackageHull kv.Key lockFile - - let packages = - allPackages - |> Seq.filter (fun p -> - match targetFilter with - | None -> true - | Some targets -> - let key = kv.Key,p - let resolvedPackage = resolved.Force().[key] - - match resolvedPackage.Settings.FrameworkRestrictions with - | Requirements.ExplicitRestriction restrictions -> - targets - |> Array.exists (fun target -> Requirements.isTargetMatchingRestrictions(restrictions, SinglePlatform target)) - | _ -> true) + // Shortcut if we already restored before + let newContents = File.ReadAllText(lockFileName.FullName) + let restoreCacheFile = Path.Combine(root, Constants.PaketFilesFolderName, Constants.RestoreHashFile) + let inline isEarlyExit () = + // We ignore our check when we do a partial restore, this way we can + // fixup project specific changes (like an additional target framework or a changed references file) + // We could still skip the actual "restore" work, but that is left as an exercise for the interesting reader. + if targetFrameworks = None && projectFile = None && referencesFileNames = [] && File.Exists restoreCacheFile then + let oldContents = File.ReadAllText(restoreCacheFile) + oldContents = newContents + else false + + if isEarlyExit () then + tracefn "Last restore is still up 2 date." + else + let dependenciesFile = DependenciesFile.ReadFromFile(dependenciesFileName) + + let targetFilter = + targetFrameworks + |> Option.map (fun s -> s.Split(';') |> Array.map FrameworkDetection.Extract |> Array.choose id) + + let lockFile,localFile,hasLocalFile = + let lockFile = LockFile.LoadFrom(lockFileName.FullName) + if not localFileName.Exists then + lockFile,LocalFile.empty,false + else + let localFile = + LocalFile.readFile localFileName.FullName + |> returnOrFail + LocalFile.overrideLockFile localFile lockFile,localFile,true + + if not hasLocalFile && not ignoreChecks then + let hasAnyChanges,nugetChanges,remoteFilechanges,hasChanges = DependencyChangeDetection.GetChanges(dependenciesFile,lockFile,false) + + let checkResponse = if failOnChecks then failwithf else traceWarnfn + if hasAnyChanges then + checkResponse "paket.dependencies and paket.lock are out of sync in %s.%sPlease run 'paket install' or 'paket update' to recompute the paket.lock file." lockFileName.Directory.FullName Environment.NewLine + for (group, package, changes) in nugetChanges do + traceWarnfn "Changes were detected for %s/%s" (group.ToString()) (package.ToString()) + for change in changes do + traceWarnfn " - %A" change + + let groups = + match group with + | None -> lockFile.Groups + | Some groupName -> + match lockFile.Groups |> Map.tryFind groupName with + | None -> failwithf "The group %O was not found in the paket.lock file." groupName + | Some group -> [groupName,group] |> Map.ofList + + let resolved = lazy (lockFile.GetGroupedResolution()) + + let referencesFileNames = + match projectFile with + | Some projectFileName -> + let projectFile = ProjectFile.LoadFromFile projectFileName + + let referencesFile = RestoreNewSdkProject dependenciesFile lockFile resolved groups projectFile + + [referencesFile.FileName] + | None -> + if referencesFileNames = [] then + // Restore all projects + ProjectFile.FindAllProjects root + |> Seq.filter (fun proj -> proj.GetToolsVersion() >= 15.0) + |> Seq.iter (fun proj -> + RestoreNewSdkProject dependenciesFile lockFile resolved groups proj + |> ignore) + + referencesFileNames + + let tasks = + groups + |> Seq.map (fun kv -> + let allPackages = + if List.isEmpty referencesFileNames then + kv.Value.Resolution + |> Seq.map (fun kv -> kv.Key) + else + referencesFileNames + |> List.toSeq + |> computePackageHull kv.Key lockFile + + let packages = + allPackages + |> Seq.filter (fun p -> + match targetFilter with + | None -> true + | Some targets -> + let key = kv.Key,p + let resolvedPackage = resolved.Force().[key] + + match resolvedPackage.Settings.FrameworkRestrictions with + | Requirements.ExplicitRestriction restrictions -> + targets + |> Array.exists (fun target -> Requirements.isTargetMatchingRestrictions(restrictions, SinglePlatform target)) + | _ -> true) - match dependenciesFile.Groups |> Map.tryFind kv.Value.Name with - | None -> - failwithf - "The group %O was found in the %s file but not in the %s file. Please run \"paket install\" again." - kv.Value - Constants.LockFileName - Constants.DependenciesFileName - | Some depFileGroup -> - let packages = Set.ofSeq packages - let overriden = - packages - |> Set.filter (fun p -> LocalFile.overrides localFile (p,depFileGroup.Name)) - - restore(alternativeProjectRoot, root, kv.Key, depFileGroup.Sources, depFileGroup.Caches, force, lockFile, packages, overriden)) - |> Seq.toArray + match dependenciesFile.Groups |> Map.tryFind kv.Value.Name with + | None -> + failwithf + "The group %O was found in the %s file but not in the %s file. Please run \"paket install\" again." + kv.Value + Constants.LockFileName + Constants.DependenciesFileName + | Some depFileGroup -> + let packages = Set.ofSeq packages + let overriden = + packages + |> Set.filter (fun p -> LocalFile.overrides localFile (p,depFileGroup.Name)) + + restore(alternativeProjectRoot, root, kv.Key, depFileGroup.Sources, depFileGroup.Caches, force, lockFile, packages, overriden)) + |> Seq.toArray - RunInLockedAccessMode( - root, - (fun () -> - for task in tasks do - task - |> Async.RunSynchronously - |> ignore - - CreateScriptsForGroups dependenciesFile lockFile groups)) \ No newline at end of file + RunInLockedAccessMode( + root, + (fun () -> + for task in tasks do + task + |> Async.RunSynchronously + |> ignore + + CreateScriptsForGroups dependenciesFile lockFile groups + if targetFrameworks = None && projectFile = None && referencesFileNames = [] then + File.WriteAllText(restoreCacheFile, newContents))) \ No newline at end of file diff --git a/src/Paket.Core/PackageManagement/NugetConvert.fs b/src/Paket.Core/PackageManagement/NugetConvert.fs index 2c4050723c..48360028b5 100644 --- a/src/Paket.Core/PackageManagement/NugetConvert.fs +++ b/src/Paket.Core/PackageManagement/NugetConvert.fs @@ -458,7 +458,7 @@ let convertR rootDirectory force credsMigrationMode = trial { return! createResult(rootDirectory, nugetEnv, credsMigrationMode) } -let replaceNuGetWithPaket initAutoRestore installAfter fromBootstrapper result = +let replaceNuGetWithPaket initAutoRestore installAfter result = let remove (fi : FileInfo) = tracefn "Removing %s" fi.FullName fi.Delete() @@ -497,7 +497,7 @@ let replaceNuGetWithPaket initAutoRestore installAfter fromBootstrapper result = if initAutoRestore && (autoVSPackageRestore || result.NuGetEnv.NuGetTargets.IsSome) then try - VSIntegration.TurnOnAutoRestore fromBootstrapper result.PaketEnv |> returnOrFail + VSIntegration.TurnOnAutoRestore result.PaketEnv |> returnOrFail with | exn -> traceWarnfn "Could not enable auto restore%sMessage: %s" Environment.NewLine exn.Message diff --git a/src/Paket.Core/PackageManagement/VSIntegration.fs b/src/Paket.Core/PackageManagement/VSIntegration.fs index 8f60ddce18..35cb30cdb6 100644 --- a/src/Paket.Core/PackageManagement/VSIntegration.fs +++ b/src/Paket.Core/PackageManagement/VSIntegration.fs @@ -27,7 +27,7 @@ let TurnOffAutoRestore environment = } /// Activates the Visual Studio NuGet autorestore feature in all projects -let TurnOnAutoRestore fromBootstrapper environment = +let TurnOnAutoRestore environment = let exeDir = Path.Combine(environment.RootDirectory.FullName, Constants.PaketFolderName) trial { diff --git a/src/Paket.Core/PublicAPI.fs b/src/Paket.Core/PublicAPI.fs index 66698088a3..c1793d20f2 100644 --- a/src/Paket.Core/PublicAPI.fs +++ b/src/Paket.Core/PublicAPI.fs @@ -70,10 +70,7 @@ type Dependencies(dependenciesFileName: string) = static member Init() = Dependencies.Init(Directory.GetCurrentDirectory()) /// Initialize paket.dependencies file in the given directory - static member Init(directory) = Dependencies.Init(directory,false) - - /// Initialize paket.dependencies file in the given directory - static member Init(directory,fromBootstrapper) = + static member Init(directory) = let directory = DirectoryInfo(directory) RunInLockedAccessMode( @@ -84,16 +81,10 @@ type Dependencies(dependenciesFileName: string) = ) let deps = Dependencies.Locate() - deps.DownloadLatestBootstrapper(fromBootstrapper) + deps.DownloadLatestBootstrapper() /// Converts the solution from NuGet to Paket. static member ConvertFromNuget(force: bool,installAfter: bool, initAutoRestore: bool,credsMigrationMode: string option, ?directory: DirectoryInfo) : unit = - match directory with - | Some d -> Dependencies.ConvertFromNuget(force, installAfter, initAutoRestore, credsMigrationMode, false, d) - | None -> Dependencies.ConvertFromNuget(force, installAfter, initAutoRestore, credsMigrationMode, false) - - /// Converts the solution from NuGet to Paket. - static member ConvertFromNuget(force: bool,installAfter: bool, initAutoRestore: bool,credsMigrationMode: string option, fromBootstrapper, ?directory: DirectoryInfo) : unit = let dir = defaultArg directory (DirectoryInfo(Directory.GetCurrentDirectory())) let rootDirectory = dir @@ -102,7 +93,7 @@ type Dependencies(dependenciesFileName: string) = fun () -> NuGetConvert.convertR rootDirectory force credsMigrationMode |> returnOrFail - |> NuGetConvert.replaceNuGetWithPaket initAutoRestore installAfter fromBootstrapper + |> NuGetConvert.replaceNuGetWithPaket initAutoRestore installAfter ) /// Converts the current package dependency graph to the simplest dependency graph. @@ -350,12 +341,8 @@ type Dependencies(dependenciesFileName: string) = |> this.Process |> List.map (fun (g, p,_,newVersion) -> g.ToString(),p.ToString(),newVersion) - /// Downloads the latest paket.bootstrapper into the .paket folder. - member this.DownloadLatestBootstrapper() : unit = - this.DownloadLatestBootstrapper(false) - /// Downloads the latest paket.bootstrapper into the .paket folder and try to rename it to paket.exe in order to activate magic mode. - member this.DownloadLatestBootstrapper(fromBootstrapper) : unit = + member this.DownloadLatestBootstrapper() : unit = RunInLockedAccessMode( this.RootPath, fun () -> @@ -370,14 +357,10 @@ type Dependencies(dependenciesFileName: string) = | _ ->()) /// Pulls new paket.targets and bootstrapper and puts them into .paket folder. - member this.TurnOnAutoRestore(fromBootstrapper: bool): unit = + member this.TurnOnAutoRestore(): unit = RunInLockedAccessMode( this.RootPath, - fun () -> VSIntegration.TurnOnAutoRestore fromBootstrapper |> this.Process) - - /// Pulls new paket.targets and bootstrapper and puts them into .paket folder. - member this.TurnOnAutoRestore(): unit = - this.TurnOnAutoRestore(false) + fun () -> VSIntegration.TurnOnAutoRestore |> this.Process) /// Removes paket.targets file and Import section from project files. member this.TurnOffAutoRestore(): unit = @@ -711,6 +694,7 @@ type Dependencies(dependenciesFileName: string) = /// Fix the transitive references in a list of generated .nuspec files + [] static member FixNuspecs (referencesFile:string, nuspecFileList:string list) = for nuspecFile in nuspecFileList do @@ -752,6 +736,42 @@ type Dependencies(dependenciesFileName: string) = use fileStream = File.Open (nuspecFile, FileMode.Create) doc.Save fileStream + static member FixNuspecs (referencesFile:ReferencesFile, nuspecFileList:string list) = + + for nuspecFile in nuspecFileList do + if not (File.Exists nuspecFile) then + failwithf "Specified file '%s' does not exist." nuspecFile + + let directDeps = + referencesFile.Groups + |> Seq.collect (fun kv -> kv.Value.NugetPackages) + |> Seq.map (fun i -> i.Name) + |> Set.ofSeq + for nuspecFile in nuspecFileList do + let nuspecText = File.ReadAllText nuspecFile + + let doc = + try let doc = Xml.XmlDocument() in doc.LoadXml nuspecText + doc + with exn -> raise <| Exception(sprintf "Could not parse nuspec file '%s'." nuspecFile, exn) + + let rec traverse (parent:XmlNode) = + let nodesToRemove = ResizeArray() + for node in parent.ChildNodes do + if node.Name = "dependency" then + let packageName = + match node.Attributes.["id"] with null -> "" | x -> x.InnerText + + if not (directDeps.Contains (PackageName packageName)) then + nodesToRemove.Add node |> ignore + + if nodesToRemove.Count = 0 then + for node in parent.ChildNodes do traverse node + else + for node in nodesToRemove do parent.RemoveChild node |> ignore + traverse doc + use fileStream = File.Open (nuspecFile, FileMode.Create) + doc.Save fileStream module PublicAPI = /// Takes a version string formatted for Semantic Versioning and parses it diff --git a/src/Paket/Commands.fs b/src/Paket/Commands.fs index 470da354a9..ba43ea5b13 100644 --- a/src/Paket/Commands.fs +++ b/src/Paket/Commands.fs @@ -357,13 +357,15 @@ with type FixNuspecsArgs = | [] Files of nuspecPaths:string list - | [] ReferencesFile of referencePath:string + | [] ReferencesFile of referencePath:string + | [] ProjectFile of referencePath:string with interface IArgParserTemplate with member this.Usage = match this with | Files _ -> ".nuspec files to fix transitive dependencies within" | ReferencesFile _ -> "paket.references to use" + | ProjectFile _ -> "the proejct file to use" type GenerateNuspecArgs = | [] Project of project:string diff --git a/src/Paket/Paket.fsproj b/src/Paket/Paket.fsproj index 33dc78b6cc..5ba8fd26b8 100644 --- a/src/Paket/Paket.fsproj +++ b/src/Paket/Paket.fsproj @@ -31,8 +31,8 @@ Project paket.exe Project - update - C:\proj\testing\testpaketfailure\ + fix-nuspecs files src/Paket.Core.preview3/obj/Paket.Core.1.0.0.nuspec project-file src/Paket.Core.preview3/Paket.Core.fsproj + C:\proj\Paket true @@ -42,8 +42,8 @@ 3 - install -v - D:\temp\test + fix-nuspecs files src/Paket.Core.preview3/obj/Paket.Core.1.0.0.nuspec project-file src/Paket.Core.preview3/Paket.Core.fsproj + C:\proj\Paket 14.0 diff --git a/src/Paket/Program.fs b/src/Paket/Program.fs index 3e3d2fd230..85ac10d0d5 100644 --- a/src/Paket/Program.fs +++ b/src/Paket/Program.fs @@ -22,11 +22,11 @@ type PaketExiter() = tracen msg ; exit 0 else traceError msg ; exit 1 -let processWithValidation silent validateF commandF (result : ParseResults<'T>) = +let processWithValidationEx printUsage silent validateF commandF result = if not <| validateF result then traceError "Command was:" traceError (" " + String.Join(" ",Environment.GetCommandLineArgs())) - result.Parser.PrintUsage() |> traceError + printUsage result #if NETCOREAPP1_0 // Environment.ExitCode not supported in netcoreapp1.0 @@ -104,6 +104,10 @@ let processWithValidation silent validateF commandF (result : ParseResults<'T>) if not verbose && omitted > 0 then traceWarnfn "Paket omitted '%d' warnings similar to the ones above. You can see them in verbose mode" omitted + +let processWithValidation silent validateF commandF (result : ParseResults<'T>) = + processWithValidationEx (fun (r:ParseResults<'T>) -> r.Parser.PrintUsage() |> traceError) silent validateF commandF result + let processCommand silent commandF result = processWithValidation silent (fun _ -> true) commandF result @@ -235,12 +239,12 @@ let config (results : ParseResults<_>) = let validateAutoRestore (results : ParseResults<_>) = results.GetAllResults().Length = 1 -let autoRestore (fromBootstrapper:bool) (results : ParseResults<_>) = +let autoRestore (results : ParseResults<_>) = match results.GetResult <@ Flags @> with - | On -> Dependencies.Locate().TurnOnAutoRestore(fromBootstrapper) + | On -> Dependencies.Locate().TurnOnAutoRestore() | Off -> Dependencies.Locate().TurnOffAutoRestore() -let convert (fromBootstrapper:bool) (results : ParseResults<_>) = +let convert (results : ParseResults<_>) = let force = results.Contains <@ ConvertFromNugetArgs.Force @> let noInstall = results.Contains <@ ConvertFromNugetArgs.No_Install @> let noAutoRestore = results.Contains <@ ConvertFromNugetArgs.No_Auto_Restore @> @@ -249,7 +253,7 @@ let convert (fromBootstrapper:bool) (results : ParseResults<_>) = results.TryGetResult <@ ConvertFromNugetArgs.Migrate_Credentials @>) |> legacyOption results (ReplaceArgument("--migrate-credentials", "--creds-migration")) - Dependencies.ConvertFromNuget(force, noInstall |> not, noAutoRestore |> not, credsMigrationMode, fromBootstrapper=fromBootstrapper) + Dependencies.ConvertFromNuget(force, noInstall |> not, noAutoRestore |> not, credsMigrationMode) let findRefs (results : ParseResults<_>) = let packages = @@ -266,8 +270,8 @@ let findRefs (results : ParseResults<_>) = packages |> List.map (fun p -> group,p) |> Dependencies.Locate().ShowReferencesFor -let init (fromBootstrapper:bool) (results : ParseResults) = - Dependencies.Init(Directory.GetCurrentDirectory(),fromBootstrapper) +let init (results : ParseResults) = + Dependencies.Init(Directory.GetCurrentDirectory()) let clearCache (results : ParseResults) = Dependencies.ClearCache() @@ -552,13 +556,27 @@ let findPackages silent (results : ParseResults<_>) = | Some searchText -> searchAndPrint searchText +#nowarn "44" // because FixNuspecs is deprecated and we have warnaserror + let fixNuspecs silent (results : ParseResults<_>) = - let referenceFile = results.GetResult <@ FixNuspecsArgs.ReferencesFile @> - let nuspecFiles = + let nuspecFiles = results.GetResult <@ FixNuspecsArgs.Files @> |> List.collect (fun s -> s.Split([|';'|], StringSplitOptions.RemoveEmptyEntries) |> Array.toList) - Dependencies.FixNuspecs (referenceFile, nuspecFiles) + match results.TryGetResult <@ FixNuspecsArgs.ProjectFile @> with + | Some projectFile -> + let projectFile = Paket.ProjectFile.LoadFromFile(projectFile) + let refFile = RestoreProcess.FindOrCreateReferencesFile projectFile + Dependencies.FixNuspecs (refFile, nuspecFiles) + | None -> + match results.TryGetResult <@ FixNuspecsArgs.ReferencesFile @> with + | Some referenceFile -> + traceWarnfn "using the references-file argument is obsolete, please use project-file instead" + + Dependencies.FixNuspecs (referenceFile, nuspecFiles) + | None -> failwithf "%s" (results.Parser.PrintUsage()) + + // For Backwards compatibility let fixNuspec silent (results : ParseResults<_>) = @@ -700,6 +718,45 @@ let why (results: ParseResults) = Why.ohWhy(packageName, directDeps, lockFile, groupName, results.Parser.PrintUsage(), options) +let handleCommand silent command = + match command with + | Add r -> processCommand silent add r + | ClearCache r -> processCommand silent clearCache r + | Config r -> processWithValidation silent validateConfig config r + | ConvertFromNuget r -> processCommand silent convert r + | FindRefs r -> processCommand silent findRefs r + | Init r -> processCommand silent (init) r + | AutoRestore r -> processWithValidation silent validateAutoRestore autoRestore r + | Install r -> processCommand silent install r + | Outdated r -> processCommand silent outdated r + | Remove r -> processCommand silent remove r + | Restore r -> processCommand silent restore r + | Simplify r -> processCommand silent simplify r + | Update r -> processCommand silent update r + | FindPackages r -> processCommand silent (findPackages silent) r + | FindPackageVersions r -> processCommand silent findPackageVersions r + | FixNuspec r -> + warnObsolete (ReplaceArgument("fix-nuspec", "fix-nuspecs")) + processCommand silent (fixNuspec silent) r + | FixNuspecs r -> processCommand silent (fixNuspecs silent) r + | ShowInstalledPackages r -> processCommand silent showInstalledPackages r + | ShowGroups r -> processCommand silent showGroups r + | Pack r -> processCommand silent pack r + | Push r -> processCommand silent (push AssemblyVersionInformation.AssemblyInformationalVersion) r + | GenerateIncludeScripts r -> + warnObsolete (ReplaceArgument("generate-include-scripts", "generate-load-scripts")) + processCommand silent generateLoadScripts r + | GenerateLoadScripts r -> processCommand silent generateLoadScripts r + | GenerateNuspec r -> processCommand silent generateNuspec r + | Why r -> processCommand silent why r + // global options; list here in order to maintain compiler warnings + // in case of new subcommands added + | Verbose + | Silent + | From_Bootstrapper + | Version + | Log_File _ -> failwithf "internal error: this code should never be reached." + let main() = let resolution = Environment.GetEnvironmentVariable ("PAKET_DISABLE_RUNTIME_RESOLUTION") if System.String.IsNullOrEmpty resolution then @@ -708,6 +765,12 @@ let main() = let paketVersion = AssemblyVersionInformation.AssemblyInformationalVersion try + let args = Environment.GetCommandLineArgs() + match args with + | [| "restore" |] | [| "--from-bootstrapper"; "restore" |] -> + // fast restore route, see https://github.com/fsprojects/Argu/issues/90 + processWithValidationEx ignore false (fun _ -> true) (fun _ -> Dependencies.Locate().Restore()) () + | _ -> let parser = ArgumentParser.Create(programName = "paket", helpTextMessage = sprintf "Paket version %s%sHelp was requested:" paketVersion Environment.NewLine, errorHandler = new PaketExiter()) @@ -720,8 +783,6 @@ let main() = if results.Contains <@ Verbose @> then Logging.verbose <- true - let fromBootstrapper = results.Contains <@ From_Bootstrapper @> - let version = results.Contains <@ Version @> if not version then @@ -730,44 +791,7 @@ let main() = | Some lf -> setLogFile lf | None -> null - match results.GetSubCommand() with - | Add r -> processCommand silent add r - | ClearCache r -> processCommand silent clearCache r - | Config r -> processWithValidation silent validateConfig config r - | ConvertFromNuget r -> processCommand silent (convert fromBootstrapper) r - | FindRefs r -> processCommand silent findRefs r - | Init r -> processCommand silent (init fromBootstrapper) r - | AutoRestore r -> processWithValidation silent validateAutoRestore (autoRestore fromBootstrapper) r - | Install r -> processCommand silent install r - | Outdated r -> processCommand silent outdated r - | Remove r -> processCommand silent remove r - | Restore r -> processCommand silent restore r - | Simplify r -> processCommand silent simplify r - | Update r -> processCommand silent update r - | FindPackages r -> processCommand silent (findPackages silent) r - | FindPackageVersions r -> processCommand silent findPackageVersions r - | FixNuspec r -> - warnObsolete (ReplaceArgument("fix-nuspec", "fix-nuspecs")) - processCommand silent (fixNuspec silent) r - | FixNuspecs r -> processCommand silent (fixNuspecs silent) r - | ShowInstalledPackages r -> processCommand silent showInstalledPackages r - | ShowGroups r -> processCommand silent showGroups r - | Pack r -> processCommand silent pack r - | Push r -> processCommand silent (push paketVersion) r - | GenerateIncludeScripts r -> - warnObsolete (ReplaceArgument("generate-include-scripts", "generate-load-scripts")) - processCommand silent generateLoadScripts r - | GenerateLoadScripts r -> processCommand silent generateLoadScripts r - | GenerateNuspec r -> processCommand silent generateNuspec r - | Why r -> processCommand silent why r - // global options; list here in order to maintain compiler warnings - // in case of new subcommands added - | Verbose - | Silent - | From_Bootstrapper - | Version - | Log_File _ -> failwithf "internal error: this code should never be reached." - + handleCommand silent (results.GetSubCommand()) with | exn when not (exn :? System.NullReferenceException) -> #if NETCOREAPP1_0 diff --git a/tools/tools.fsproj b/tools/tools.fsproj index 76a5c9cde3..05b4e48a2b 100644 --- a/tools/tools.fsproj +++ b/tools/tools.fsproj @@ -1,4 +1,4 @@ - + netstandard1.6