diff --git a/src/nimblepkg/download.nim b/src/nimblepkg/download.nim index 9db9bba1..0535fe13 100644 --- a/src/nimblepkg/download.nim +++ b/src/nimblepkg/download.nim @@ -19,7 +19,7 @@ proc updateSubmodules(dir: string) = discard tryDoCmdEx( &"git -C {dir} submodule update --init --recursive --depth 1") -proc doCheckout(meth: DownloadMethod, downloadDir, branch: string) = +proc doCheckout*(meth: DownloadMethod, downloadDir, branch: string) = case meth of DownloadMethod.git: # Force is used here because local changes may appear straight after a clone @@ -46,7 +46,15 @@ proc doClone(meth: DownloadMethod, url, downloadDir: string, branch = "", branchArg = if branch == "": "" else: &"-b {branch}" discard tryDoCmdEx(&"hg clone {tipArg} {branchArg} {url} {downloadDir}") -proc getTagsList(dir: string, meth: DownloadMethod): seq[string] = +proc gitFetchTags*(repoDir: string, downloadMethod: DownloadMethod) = + case downloadMethod: + of DownloadMethod.git: + tryDoCmdEx(&"git -C {repoDir} fetch --tags") + of DownloadMethod.hg: + assert false, "hg not supported" + + +proc getTagsList*(dir: string, meth: DownloadMethod): seq[string] = var output: string cd dir: case meth diff --git a/src/nimblepkg/nimblesat.nim b/src/nimblepkg/nimblesat.nim index d7ec016d..3aae89be 100644 --- a/src/nimblepkg/nimblesat.nim +++ b/src/nimblepkg/nimblesat.nim @@ -1,6 +1,6 @@ import sat/[sat, satvars] import version, packageinfotypes, download, packageinfo, packageparser, options, - sha1hashes, tools, downloadnim + sha1hashes, tools, downloadnim, cli import std/[tables, sequtils, algorithm, sets, strutils, options, strformat, os] @@ -315,10 +315,33 @@ proc findMinimalFailingSet*(g: var DepGraph): tuple[failingSet: seq[PkgTuple], o (minimalFailingSet, output) -#It may be better to just use result here -proc solve*(g: var DepGraph; f: Form, packages: var Table[string, Version], output: var string): bool = +proc filterSatisfiableDeps(g: DepGraph, node: Dependency): seq[DependencyVersion] = + ## Returns a sequence of versions from the node that have satisfiable dependencies + result = @[] + for v in node.versions: + let reqs = g.reqs[v.req].deps + var hasUnsatisfiableDep = false + for req in reqs: + let depIdx = findDependencyForDep(g, req.name) + if depIdx >= 0: + var canSatisfy = false + for depVer in g.nodes[depIdx].versions: + if depVer.version.withinRange(req.ver): + canSatisfy = true + break + if not canSatisfy: + hasUnsatisfiableDep = true + break + if not hasUnsatisfiableDep: + result.add(v) + +const MaxSolverRetries = 100 + +proc solve*(g: var DepGraph; f: Form, packages: var Table[string, Version], output: var string, + retryCount = 0): bool = let m = f.idgen var s = createSolution(m) + if satisfiable(f.f, s): for n in mitems g.nodes: if n.isRoot: n.active = true @@ -330,18 +353,33 @@ proc solve*(g: var DepGraph; f: Form, packages: var Table[string, Version], outp g.nodes[idx].activeVersion = m.index for n in items g.nodes: - for v in items(n.versions): - let item = f.mapping[v.v] - if s.isTrue(v.v): - packages[item.pkg] = item.version - output.add &"item.pkg [x] {toString item} \n" - else: - output.add &"item.pkg [ ] {toString item} \n" - # echo output - true + for v in items(n.versions): + let item = f.mapping[v.v] + if s.isTrue(v.v): + packages[item.pkg] = item.version + output.add &"item.pkg [x] {toString item} \n" + else: + output.add &"item.pkg [ ] {toString item} \n" + return true else: let (failingSet, errorMsg) = findMinimalFailingSet(g) + if retryCount >= MaxSolverRetries: + output = &"Max retry attempts ({MaxSolverRetries}) exceeded while trying to resolve dependencies \n" + output.add errorMsg + return false + if failingSet.len > 0: + var newGraph = g + for pkg in failingSet: + let idx = findDependencyForDep(newGraph, pkg.name) + if idx >= 0: + # echo "Retry #", retryCount + 1, ": Checking package ", pkg.name, " version ", pkg.ver + let newVersions = filterSatisfiableDeps(newGraph, newGraph.nodes[idx]) + if newVersions.len > 0 and newVersions != newGraph.nodes[idx].versions: + newGraph.nodes[idx].versions = newVersions + let newForm = toFormular(newGraph) + return solve(newGraph, newForm, packages, output, retryCount + 1) + output = errorMsg else: output = generateUnsatisfiableMessage(g, f, s) @@ -362,7 +400,12 @@ proc getSolvedPackages*(pkgVersionTable: Table[string, PackageVersions], output: for ver in p.versions.items: for dep, q in items graph.reqs[ver.req].deps: if dep notin graph.packageToDependency: + #debug print. show all packacges in the graph output.add &"Dependency {dep} not found in the graph \n" + for k, v in pkgVersionTable: + output.add &"Package {k} \n" + for v in v.versions: + output.add &"\t \t Version {v.version} requires: {v.requires} \n" return newSeq[SolvedPackage]() let form = toFormular(graph) @@ -384,27 +427,55 @@ proc getSolvedPackages*(pkgVersionTable: Table[string, PackageVersions], output: proc getCacheDownloadDir*(url: string, ver: VersionRange, options: Options): string = options.pkgCachePath / getDownloadDirName(url, ver, notSetSha1Hash) -proc downloadPkInfoForPv*(pv: PkgTuple, options: Options): PackageInfo = +proc downloadPkgFromUrl*(pv: PkgTuple, options: Options): (DownloadPkgResult, DownloadMethod) = let (meth, url, metadata) = - getDownloadInfo(pv, options, doPrompt = false, ignorePackageCache = false) + getDownloadInfo(pv, options, doPrompt = false, ignorePackageCache = false) let subdir = metadata.getOrDefault("subdir") let downloadDir = getCacheDownloadDir(url, pv.ver, options) - let res = - downloadPkg(url, pv.ver, meth, subdir, options, - downloadDir, vcsRevision = notSetSha1Hash) - return getPkgInfo(res.dir, options) + let downloadRes = downloadPkg(url, pv.ver, meth, subdir, options, + downloadDir, vcsRevision = notSetSha1Hash) + (downloadRes, meth) + +proc downloadPkInfoForPv*(pv: PkgTuple, options: Options): PackageInfo = + downloadPkgFromUrl(pv, options)[0].dir.getPkgInfo(options) proc getAllNimReleases(options: Options): seq[PackageMinimalInfo] = let releases = getOfficialReleases(options) for release in releases: result.add PackageMinimalInfo(name: "nim", version: release) + +proc getPackageMinimalVersionsFromRepo*(repoDir, pkgName: string, downloadMethod: DownloadMethod, options: Options): seq[PackageMinimalInfo] = + #This is expensive. We need to cache it. Potentially it could be also run in parallel + # echo &"Discovering version for {pkgName}" + gitFetchTags(repoDir, downloadMethod) + #First package must be the current one + result.add getPkgInfo(repoDir, options).getMinimalInfo(options) + let tags = getTagsList(repoDir, downloadMethod).getVersionList() + var checkedTags = 0 + for (ver, tag) in tags.pairs: + if options.maxTaggedVersions > 0 and checkedTags >= options.maxTaggedVersions: + # echo &"Tag limit reached for {pkgName}" + break + inc checkedTags + #For each version, we need to parse the requires so we need to checkout and initialize the repo + try: + doCheckout(downloadMethod, repoDir, tag) + let nimbleFile = findNimbleFile(repoDir, true, options) + let pkgInfo = getPkgInfoFromFile(nimbleFile, options, useCache=false) + let minimalInfo = pkgInfo.getMinimalInfo(options) + result.addUnique minimalInfo + except CatchableError as e: + displayWarning(&"Error reading tag {tag}: for package {pkgName}. This may not be relevant as it could be an old version of the package. \n {e.msg}", HighPriority) proc downloadMinimalPackage*(pv: PkgTuple, options: Options): seq[PackageMinimalInfo] = if pv.name == "": return newSeq[PackageMinimalInfo]() if pv.isNim and not options.disableNimBinaries: return getAllNimReleases(options) - - let pkgInfo = downloadPkInfoForPv(pv, options) - return @[pkgInfo.getMinimalInfo(options)] + if pv.ver.kind in [verSpecial, verEq]: #if special or equal, we dont retrieve more versions as we only need one. + result = @[downloadPkInfoForPv(pv, options).getMinimalInfo(options)] + else: + let (downloadRes, downloadMeth) = downloadPkgFromUrl(pv, options) + result = getPackageMinimalVersionsFromRepo(downloadRes.dir, pv.name, downloadMeth, options) + # echo "Downloading minimal package for ", pv.name, " ", $pv.ver, result proc fillPackageTableFromPreferred*(packages: var Table[string, PackageVersions], preferredPackages: seq[PackageMinimalInfo]) = for pkg in preferredPackages: @@ -417,20 +488,17 @@ proc fillPackageTableFromPreferred*(packages: var Table[string, PackageVersions] proc getInstalledMinimalPackages*(options: Options): seq[PackageMinimalInfo] = getInstalledPkgsMin(options.getPkgsDir(), options).mapIt(it.getMinimalInfo(options)) -proc collectAllVersions*(versions: var Table[string, PackageVersions], package: PackageMinimalInfo, options: Options, getMinimalPackage: GetPackageMinimal, preferredPackages: seq[PackageMinimalInfo] = newSeq[PackageMinimalInfo]()) = - ### Collects all the versions of a package and its dependencies and stores them in the versions table - ### A getMinimalPackage function is passed to get the package + +proc collectAllVersions*(versions: var Table[string, PackageVersions], package: PackageMinimalInfo, options: Options, getMinimalPackage: GetPackageMinimal, preferredPackages: seq[PackageMinimalInfo] = newSeq[PackageMinimalInfo]()) = proc getMinimalFromPreferred(pv: PkgTuple): seq[PackageMinimalInfo] = - #Before proceding to download we check if the package is in the preferred packages for pp in preferredPackages: if pp.name == pv.name and pp.version.withinRange(pv.ver): return @[pp] + # echo "Getting minimal from getMinimalPackage for ", pv.name, " ", $pv.ver getMinimalPackage(pv, options) - for pv in package.requires: - # echo "Collecting versions for ", pv.name, " and Version: ", $pv.ver, " via ", package.name - var pv = pv - if not hasVersion(versions, pv): # Not found, meaning this package-version needs to be explored + proc processRequirements(versions: var Table[string, PackageVersions], pv: PkgTuple) = + if not hasVersion(versions, pv): var pkgMins = getMinimalFromPreferred(pv) for pkgMin in pkgMins.mitems: if pv.ver.kind == verSpecial: @@ -439,8 +507,14 @@ proc collectAllVersions*(versions: var Table[string, PackageVersions], package: versions[pv.name] = PackageVersions(pkgName: pv.name, versions: @[pkgMin]) else: versions[pv.name].versions.addUnique pkgMin - #TODO Note for when implementing "enumerate all versions": do not enter in the loop until we have collected all the versions - collectAllVersions(versions, pkgMin, options, getMinimalPackage, preferredPackages) + + # Process requirements from both the package and GetMinimalPackage results + for req in pkgMin.requires: + # echo "Processing requirement: ", req.name, " ", $req.ver + processRequirements(versions, req) + + for pv in package.requires: + processRequirements(versions, pv) proc topologicalSort*(solvedPkgs: seq[SolvedPackage]): seq[SolvedPackage] = var inDegree = initTable[string, int]() @@ -510,4 +584,4 @@ proc getPackageInfo*(name: string, pkgs: seq[PackageInfo], version: Option[Versi if pkg.basicInfo.version == version.get: return some pkg else: #No version passed over first match - return some pkg \ No newline at end of file + return some pkg diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim index 92660511..a974b7fb 100644 --- a/src/nimblepkg/options.nim +++ b/src/nimblepkg/options.nim @@ -63,6 +63,7 @@ type extraRequires*: seq[PkgTuple] # extra requires parsed from the command line nimBinariesDir*: string # Directory where nim binaries are stored. Separated from nimbleDir as it can be changed by the user/tests disableNimBinaries*: bool # Whether to disable the use of nim binaries + maxTaggedVersions*: int # Maximum number of tags to check for a package when discovering versions in a local repo ActionType* = enum actionNil, actionRefresh, actionInit, actionDump, actionPublish, actionUpgrade @@ -747,7 +748,8 @@ proc initOptions*(): Options = verbosity: HighPriority, noColor: not isatty(stdout), startDir: getCurrentDir(), - nimBinariesDir: getHomeDir() / ".nimble" / "nimbinaries" + nimBinariesDir: getHomeDir() / ".nimble" / "nimbinaries", + maxTaggedVersions: 2 #TODO increase once we have a cache ) proc handleUnknownFlags(options: var Options) = diff --git a/src/nimblepkg/packageparser.nim b/src/nimblepkg/packageparser.nim index 089f0f2e..d18e236b 100644 --- a/src/nimblepkg/packageparser.nim +++ b/src/nimblepkg/packageparser.nim @@ -272,7 +272,7 @@ proc inferInstallRules(pkgInfo: var PackageInfo, options: Options) = if fileExists(pkgInfo.getRealDir() / pkgInfo.basicInfo.name.addFileExt("nim")): pkgInfo.installFiles.add(pkgInfo.basicInfo.name.addFileExt("nim")) -proc readPackageInfo(pkgInfo: var PackageInfo, nf: NimbleFile, options: Options, onlyMinimalInfo=false) = +proc readPackageInfo(pkgInfo: var PackageInfo, nf: NimbleFile, options: Options, onlyMinimalInfo=false, useCache=true) = ## Reads package info from the specified Nimble file. ## ## Attempts to read it using the "old" Nimble ini format first, if that @@ -289,7 +289,7 @@ proc readPackageInfo(pkgInfo: var PackageInfo, nf: NimbleFile, options: Options, assert fileExists(nf) # Check the cache. - if options.pkgInfoCache.hasKey(nf): + if useCache and options.pkgInfoCache.hasKey(nf): pkgInfo = options.pkgInfoCache[nf] return pkgInfo = initPackageInfo(options, nf) @@ -369,12 +369,12 @@ proc readPackageInfo(pkgInfo: var PackageInfo, nf: NimbleFile, options: Options, validatePackageInfo(pkgInfo, options) proc getPkgInfoFromFile*(file: NimbleFile, options: Options, - forValidation = false): PackageInfo = + forValidation = false, useCache = true): PackageInfo = ## Reads the specified .nimble file and returns its data as a PackageInfo ## object. Any validation errors are handled and displayed as warnings. result = initPackageInfo() try: - readPackageInfo(result, file, options) + readPackageInfo(result, file, options, useCache= useCache) except ValidationError: let exc = (ref ValidationError)(getCurrentException()) if exc.warnAll and not forValidation: diff --git a/tests/oldnimble/oldnimble.nimble b/tests/oldnimble/oldnimble.nimble new file mode 100644 index 00000000..070bebbb --- /dev/null +++ b/tests/oldnimble/oldnimble.nimble @@ -0,0 +1,13 @@ +# Package + +version = "0.1.0" +author = "jmgomez" +description = "A new awesome nimble package" +license = "MIT" +srcDir = "src" + + +# Dependencies + +requires "nim >= 2.0.11" +requires "nimble <= 0.16.2" #We know this nimble version has additional requirements (new nimble use submodules) diff --git a/tests/oldnimble/src/oldnimble.nim b/tests/oldnimble/src/oldnimble.nim new file mode 100644 index 00000000..b7a24803 --- /dev/null +++ b/tests/oldnimble/src/oldnimble.nim @@ -0,0 +1,7 @@ +# This is just an example to get you started. A typical library package +# exports the main API in this file. Note that you cannot rename this file +# but you can remove it if you wish. + +proc add*(x, y: int): int = + ## Adds two numbers together. + return x + y diff --git a/tests/oldnimble/src/oldnimble/submodule.nim b/tests/oldnimble/src/oldnimble/submodule.nim new file mode 100644 index 00000000..36fbf77d --- /dev/null +++ b/tests/oldnimble/src/oldnimble/submodule.nim @@ -0,0 +1,12 @@ +# This is just an example to get you started. Users of your library will +# import this file by writing ``import oldnimble/submodule``. Feel free to rename or +# remove this file altogether. You may create additional modules alongside +# this file as required. + +type + Submodule* = object + name*: string + +proc initSubmodule*(): Submodule = + ## Initialises a new ``Submodule`` object. + Submodule(name: "Anonymous") diff --git a/tests/oldnimble/tests/test1.nim b/tests/oldnimble/tests/test1.nim new file mode 100644 index 00000000..f8512dbe --- /dev/null +++ b/tests/oldnimble/tests/test1.nim @@ -0,0 +1,12 @@ +# This is just an example to get you started. You may wish to put all of your +# tests into a single file, or separate them into multiple `test1`, `test2` +# etc. files (better names are recommended, just make sure the name starts with +# the letter 't'). +# +# To run these tests, simply execute `nimble test`. + +import unittest + +import oldnimble +test "can add": + check add(5, 5) == 10 diff --git a/tests/tsat.nim b/tests/tsat.nim index 128e9f11..d3eb368d 100644 --- a/tests/tsat.nim +++ b/tests/tsat.nim @@ -2,11 +2,10 @@ import unittest, os import testscommon # from nimblepkg/common import cd Used in the commented tests -import std/[tables, sequtils, json, jsonutils, strutils, times, options] -import nimblepkg/[version, nimblesat, options, config] +import std/[tables, sequtils, json, jsonutils, strutils, times, options, strformat] +import nimblepkg/[version, nimblesat, options, config, download, packageinfotypes, packageinfo] from nimblepkg/common import cd - proc initFromJson*(dst: var PkgTuple, jsonNode: JsonNode, jsonPath: var string) = dst = parseRequires(jsonNode.str) @@ -92,7 +91,6 @@ suite "SAT solver": check packages["a"] == newVersion "3.0" check packages["b"] == newVersion "0.1.0" - test "solves 'Conflicting dependency resolution' #1162": let pkgVersionTable = { "a": PackageVersions(pkgName: "a", versions: @[ @@ -295,3 +293,83 @@ suite "SAT solver": check packages.len == 2 check packages["a"] == newVersion "3.0" check packages["b"] == newVersion "1.0.0" # Should pick exact version 1.0.0 despite 2.0.0 being available + + test "should be able to get all the released PackageVersions from a git local repository": + var options = initOptions() + options.maxTaggedVersions = 0 #all + options.nimBin = some options.makeNimBin("nim") + options.config.packageLists["official"] = PackageList(name: "Official", urls: @[ + "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json", + "https://nim-lang.org/nimble/packages.json" + ]) + let pv = parseRequires("nimfp >= 0.3.4") + let repoDir = pv.downloadPkgFromUrl(options)[0].dir #This is just to setup the test. We need a git dir to work on + let downloadMethod = DownloadMethod git + + let packageVersions = getPackageMinimalVersionsFromRepo(repoDir, pv[0], downloadMethod, options) + + #we know these versions are available + let availableVersions = @["0.3.4", "0.3.5", "0.3.6", "0.4.5", "0.4.4"].mapIt(newVersion(it)) + for version in availableVersions: + check version in packageVersions.mapIt(it.version) + + test "if a dependency is unsatisfable, it should fallback to the previous version of the depency when available": + let pkgVersionTable = { + "a": PackageVersions(pkgName: "a", versions: @[ + PackageMinimalInfo(name: "a", version: newVersion "3.0", requires: @[ + (name:"b", ver: parseVersionRange ">= 0.5.0") + ], isRoot: true), + ]), + "b": PackageVersions(pkgName: "b", versions: @[ + PackageMinimalInfo(name: "b", version: newVersion "0.6.0", requires: @[ + (name:"c", ver: parseVersionRange ">= 0.0.5") + ]), + PackageMinimalInfo(name: "b", version: newVersion "0.5.0", requires: @[ + + ]), + ]), + "c": PackageVersions(pkgName: "c", versions: @[ + PackageMinimalInfo(name: "c", version: newVersion "0.0.4"), + ]) + }.toTable() + + var graph = pkgVersionTable.toDepGraph() + let form = toFormular(graph) + var packages = initTable[string, Version]() + var output = "" + check solve(graph, form, packages, output) + + test "collectAllVersions should retrieve all releases of a given package": + var options = initOptions() + options.nimBin = some options.makeNimBin("nim") + options.config.packageLists["official"] = PackageList(name: "Official", urls: @[ + "https://raw.githubusercontent.com/nim-lang/packages/master/packages.json", + "https://nim-lang.org/nimble/packages.json" + ]) + let pv = parseRequires("chronos >= 4.0.0") + var pkgInfo = downloadPkInfoForPv(pv, options) + var root = pkgInfo.getMinimalInfo(options) + root.isRoot = true + var pkgVersionTable = initTable[string, PackageVersions]() + collectAllVersions(pkgVersionTable, root, options, downloadMinimalPackage) + for k, v in pkgVersionTable: + if not k.isNim: + check v.versions.len <= options.maxTaggedVersions + echo &"{k} versions {v.versions.len}" + + test "should fallback to a previous version of a dependency when is unsatisfable": + #version 0.4.5 of nimfp requires nim as `nim 0.18.0` and other deps require `nim > 0.18.0` + #version 0.4.4 tags it properly, so we test thats the one used + #i.e when maxTaggedVersions is 1 it would fail as it would use 0.4.5 + cd "wronglytaggednim": + removeDir("nimbledeps") + let (_, exitCode) = execNimble("install", "-l") + check exitCode == QuitSuccess + + test "should be able to collect all requires from old versions": + #We know this nimble version has additional requirements (new nimble use submodules) + #so if the requires are not collected we will not be able solve the package + cd "oldnimble": #0.16.2 + removeDir("nimbledeps") + let (_, exitCode) = execNimbleYes("install", "-l") + check exitCode == QuitSuccess \ No newline at end of file diff --git a/tests/wronglytaggednim/src/wronglytaggednim.nim b/tests/wronglytaggednim/src/wronglytaggednim.nim new file mode 100644 index 00000000..b7a24803 --- /dev/null +++ b/tests/wronglytaggednim/src/wronglytaggednim.nim @@ -0,0 +1,7 @@ +# This is just an example to get you started. A typical library package +# exports the main API in this file. Note that you cannot rename this file +# but you can remove it if you wish. + +proc add*(x, y: int): int = + ## Adds two numbers together. + return x + y diff --git a/tests/wronglytaggednim/src/wronglytaggednim/submodule.nim b/tests/wronglytaggednim/src/wronglytaggednim/submodule.nim new file mode 100644 index 00000000..63299f12 --- /dev/null +++ b/tests/wronglytaggednim/src/wronglytaggednim/submodule.nim @@ -0,0 +1,12 @@ +# This is just an example to get you started. Users of your library will +# import this file by writing ``import wronglytaggednim/submodule``. Feel free to rename or +# remove this file altogether. You may create additional modules alongside +# this file as required. + +type + Submodule* = object + name*: string + +proc initSubmodule*(): Submodule = + ## Initialises a new ``Submodule`` object. + Submodule(name: "Anonymous") diff --git a/tests/wronglytaggednim/tests/test1.nim b/tests/wronglytaggednim/tests/test1.nim new file mode 100644 index 00000000..3755c631 --- /dev/null +++ b/tests/wronglytaggednim/tests/test1.nim @@ -0,0 +1,12 @@ +# This is just an example to get you started. You may wish to put all of your +# tests into a single file, or separate them into multiple `test1`, `test2` +# etc. files (better names are recommended, just make sure the name starts with +# the letter 't'). +# +# To run these tests, simply execute `nimble test`. + +import unittest + +import wronglytaggednim +test "can add": + check add(5, 5) == 10 diff --git a/tests/wronglytaggednim/wronglytaggednim.nimble b/tests/wronglytaggednim/wronglytaggednim.nimble new file mode 100644 index 00000000..c9664b0a --- /dev/null +++ b/tests/wronglytaggednim/wronglytaggednim.nimble @@ -0,0 +1,12 @@ +# Package + +version = "0.1.0" +author = "jmgomez" +description = "A new awesome nimble package" +license = "MIT" +srcDir = "src" + + +# Dependencies + +requires "nim >= 0.18.0", "random >= 0.5.6", "nimfp <= 0.4.5"