Merge branch 'devel' into split-custom-pragam-macro

.github/ISSUE_TEMPLATE/bug_report.md

@@ -12,9 +12,14 @@ Function `echo` outputs the wrong string.
 ### Example
 ```nim
 echo "Hello World!"
+# This code should be a minimum reproducible example:
+# try to simplify and minimize as much as possible. If it's a compiler
+# issue, try to minimize further by removing any imports if possible.
 ```
 
 ### Current Output
+please check whether the problem still exists in git head before posting,
+see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
 ```
 Hola mundo!
 ```
@@ -29,12 +34,18 @@ Hello World!
 * In file xyz there is a call that might be the cause of it.
 
 ### Additional Information
+If it's a regression, you can help us by identifying which version introduced
+the bug, see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
+or at least try known past releases (eg `choosenim 1.2.0`).
+
+If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
+which should give more context on a compiler crash.
 
-* It was working in version a.b.c
 * Issue #abc is related, but different because of ...
 * This issue is blocking my project xyz
 
 ```
 $ nim -v
 Nim Compiler Version 0.1.2
+# make sure to include the git hash if not using a tagged release
 ```

.github/workflows/ci_docs.yml

@@ -1,20 +1,30 @@
 name: Nim Docs CI
 on:
   push:
-    # Run only on changes on these files
     paths:
-      - 'lib/**.nim'
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
       - 'doc/**.rst'
       - 'doc/nimdoc.css'
+      - 'lib/**.nim'
+      - 'nimdoc/testproject/expected/testproject.html'
+      - 'tools/dochack/dochack.nim'
+      - 'tools/kochdocs.nim'
       - '.github/workflows/ci_docs.yml'
-
   pull_request:
-    # Run only on changes on these files
+    # Run only on changes on these files.
     paths:
-      - 'lib/**.nim'
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
       - 'doc/**.rst'
+      - 'doc/nimdoc.css'
+      - 'lib/**.nim'
+      - 'nimdoc/testproject/expected/testproject.html'
+      - 'tools/dochack/dochack.nim'
+      - 'tools/kochdocs.nim'
       - '.github/workflows/ci_docs.yml'
 
+
 jobs:
   build:
     strategy:
@@ -115,13 +125,6 @@ jobs:
         shell: bash
         run: ./koch doc --git.commit:devel
 
-      - name: 'Prepare documentation for deployment'
-        if: |
-          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
-          matrix.target == 'linux'
-        shell: bash
-        run: cp -f doc/html/{overview,index}.html
-
       - name: 'Publish documentation to Github Pages'
         if: |
           github.event_name == 'push' && github.ref == 'refs/heads/devel' &&

.gitignore

@@ -92,7 +92,8 @@ megatest.nim
 # ignore debug dirs generated by dsymutil on OSX
 *.dSYM
 
-nimdoc.out.css
-
 # for `nim c -r nimdoc/tester` etc; this can be in multiple places
 htmldocs
+
+## these are not needed anymore unless checkout old older versions
+nimdoc.out.css

changelog.md

@@ -3,7 +3,9 @@
 
 
 ## Standard library additions and changes
+
 - Custom pragma values have now an API for use in macros.
+- Added `bindParams`, `bindParam` to `db_sqlite` for binding parameters into a `SqlPrepared` statement.
 - Add `tryInsert`,`insert` procs to db_* libs accept primary key column name.
 - Added `xmltree.newVerbatimText` support create `style`'s,`script`'s text.
 - `uri` adds Data URI Base64, implements RFC-2397.
@@ -61,7 +63,6 @@
 
 - Added high-level `asyncnet.sendTo` and `asyncnet.recvFrom`. UDP functionality.
 
-- `paramCount` & `paramStr` are now defined in os.nim instead of nimscript.nim for nimscript/nimble.
 - `dollars.$` now works for unsigned ints with `nim js`
 
 - Improvements to the `bitops` module, including bitslices, non-mutating versions
@@ -95,10 +96,25 @@
 - The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
 - The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.
 
-- `osproc.execCmdEx` now takes an optional `input` for stdin.
 - `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
   parameters.
 
+- Add `ssl_config` module containing lists of secure ciphers as recommended by
+  [Mozilla OpSec](https://wiki.mozilla.org/Security/Server_Side_TLS)
+
+- `net.newContext` now defaults to the list of ciphers targeting
+  ["Intermediate compatibility"](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)
+  per Mozilla's recommendation instead of `ALL`. This change should protect
+  users from the use of weak and insecure ciphers while still provides
+  adequate compatiblity with the majority of the Internet.
+
+- new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` for json serialization/deserialization of custom types.
+
+- new proc `heapqueue.find[T](heap: HeapQueue[T], x: T): int` to get index of element ``x``.
+- Add `rstgen.rstToLatex` convenience proc for `renderRstToOut` and `initRstGenerator` with `outLatex` output.
+- Add `os.normalizeExe`, eg: `koch` => `./koch`.
+
+
 ## Language changes
 - In the newruntime it is now allowed to assign discriminator field without restrictions as long as case object doesn't have custom destructor. Discriminator value doesn't have to be a constant either. If you have custom destructor for case object and you do want to freely assign discriminator fields, it is recommended to refactor object into 2 objects like this:
 
@@ -159,6 +175,8 @@ proc mydiv(a, b): int {.raises: [].} =
   The reason for this is that `DivByZeroDefect` inherits from `Defect` and
   with `--panics:on` `Defects` become unrecoverable errors.
 
+- Added `thiscall` calling convention as specified by Microsoft, mostly for hooking purpose
+
 ## Compiler changes
 
 - Specific warnings can now be turned into errors via `--warningAsError[X]:on|off`.
@@ -189,6 +207,8 @@ proc mydiv(a, b): int {.raises: [].} =
   and avoids polluting both $pwd and $projectdir. It can be used with any command.
 - `runnableExamples "-b:cpp -r:off": code` is now supported, allowing to override how an example is compiled and run,
   for example to change backend or compile only.
+- `nim doc` now outputs under `$projectPath/htmldocs` when `--outdir` is unspecified (with or without `--project`);
+  passing `--project` now automatically generates an index and enables search.
+  See [docgen](docgen.html#introduction-quick-start) for details.
 
 ## Tool changes
-

compiler/ast.nim

@@ -25,12 +25,13 @@ type
     ccInline,                 # proc should be inlined
     ccNoInline,               # proc should not be inlined
     ccFastCall,               # fastcall (pass parameters in registers)
+    ccThisCall,               # thiscall (parameters are pushed right-to-left)
     ccClosure,                # proc has a closure
     ccNoConvention            # needed for generating proper C procs sometimes
 
 const
   CallingConvToStr*: array[TCallingConvention, string] = ["", "stdcall",
-    "cdecl", "safecall", "syscall", "inline", "noinline", "fastcall",
+    "cdecl", "safecall", "syscall", "inline", "noinline", "fastcall", "thiscall",
     "closure", "noconv"]
 
 type
@@ -292,6 +293,7 @@ type
     sfInjectDestructors # whether the proc needs the 'injectdestructors' transformation
     sfNeverRaises     # proc can never raise an exception, not even OverflowDefect
                       # or out-of-memory
+    sfUsedInFinallyOrExcept  # symbol is used inside an 'except' or 'finally'
 
   TSymFlags* = set[TSymFlag]
 
@@ -432,8 +434,8 @@ type
       # instantiation and prior to this it has the potential to
       # be any type.
 
-    tyOpt
-      # Builtin optional type
+    tyOptDeprecated
+      # deadcode: was `tyOpt`, Builtin optional type
 
     tyVoid
       # now different from tyEmpty, hurray!
@@ -656,7 +658,7 @@ type
     mNewString, mNewStringOfCap, mParseBiggestFloat,
     mMove, mWasMoved, mDestroy,
     mDefault, mUnown, mAccessEnv, mReset,
-    mArray, mOpenArray, mRange, mSet, mSeq, mOpt, mVarargs,
+    mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
     mOrdinal,
     mInt, mInt8, mInt16, mInt32, mInt64,
@@ -1029,6 +1031,7 @@ const
   nkFloatLiterals* = {nkFloatLit..nkFloat128Lit}
   nkLambdaKinds* = {nkLambda, nkDo}
   declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
+  routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef}
   procDefs* = nkLambdaKinds + declarativeDefs
 
   nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}
@@ -1489,7 +1492,7 @@ proc isGCedMem*(t: PType): bool {.inline.} =
            t.kind == tyProc and t.callConv == ccClosure
 
 proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) =
-  const HaveTheirOwnEmpty = {tySequence, tyOpt, tySet, tyPtr, tyRef, tyProc}
+  const HaveTheirOwnEmpty = {tySequence, tySet, tyPtr, tyRef, tyProc}
   owner.flags = owner.flags + (elem.flags * {tfHasMeta, tfTriggersCompileTime})
   if tfNotNil in elem.flags:
     if owner.kind in {tyGenericInst, tyGenericBody, tyGenericInvocation}:
@@ -1502,7 +1505,7 @@ proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) =
   if mask != {} and propagateHasAsgn:
     let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink})
     if o2.kind in {tyTuple, tyObject, tyArray,
-                   tySequence, tyOpt, tySet, tyDistinct, tyOpenArray, tyVarargs}:
+                   tySequence, tySet, tyDistinct, tyOpenArray, tyVarargs}:
       o2.flags.incl mask
       owner.flags.incl mask
 

compiler/ccgcalls.nim

@@ -22,14 +22,20 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
     result = canRaiseConservative(n)
 
 proc preventNrvo(p: BProc; le, ri: PNode): bool =
-  proc locationEscapes(p: BProc; le: PNode): bool =
+  proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
     var n = le
     while true:
       # do NOT follow nkHiddenDeref here!
       case n.kind
       of nkSym:
         # we don't own the location so it escapes:
-        return n.sym.owner != p.prc
+        if n.sym.owner != p.prc:
+          return true
+        elif inTryStmt and sfUsedInFinallyOrExcept in n.sym.flags:
+          # it is also an observable store if the location is used
+          # in 'except' or 'finally'
+          return true
+        return false
       of nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
           nkCheckedFieldExpr:
         n = n[0]
@@ -43,8 +49,11 @@ proc preventNrvo(p: BProc; le, ri: PNode): bool =
     for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
-    return canRaiseDisp(p, ri[0]) and
-        (p.nestedTryStmts.len > 0 or locationEscapes(p, le))
+    # we use the weaker 'canRaise' here in order to prevent too many
+    # annoying warnings, see #14514
+    if canRaise(ri[0]) and
+        locationEscapes(p, le, p.nestedTryStmts.len > 0):
+      message(p.config, le.info, warnObservableStores, $le)
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags

compiler/ccgexprs.nim

@@ -141,6 +141,7 @@ proc genSetNode(p: BProc, n: PNode): Rope =
     result = genRawSetData(cs, size)
 
 proc getStorageLoc(n: PNode): TStorageLoc =
+  ## deadcode
   case n.kind
   of nkSym:
     case n.sym.kind

compiler/ccgtypes.nim

@@ -238,7 +238,7 @@ const
     "N_STDCALL", "N_CDECL", "N_SAFECALL",
     "N_SYSCALL", # this is probably not correct for all platforms,
                  # but one can #define it to what one wants
-    "N_INLINE", "N_NOINLINE", "N_FASTCALL", "N_CLOSURE", "N_NOCONV"]
+    "N_INLINE", "N_NOINLINE", "N_FASTCALL", "N_THISCALL", "N_CLOSURE", "N_NOCONV"]
 
 proc cacheGetType(tab: TypeCache; sig: SigHash): Rope =
   # returns nil if we need to declare this type
@@ -268,19 +268,18 @@ proc ccgIntroducedPtr(conf: ConfigRef; s: PSym, retType: PType): bool =
       result = true
     elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
       result = true           # requested anyway
-    elif retType != nil and retType.kind == tyLent:
-      result = true
     elif (tfFinal in pt.flags) and (pt[0] == nil):
       result = false          # no need, because no subtyping possible
     else:
       result = true           # ordinary objects are always passed by reference,
                               # otherwise casting doesn't work
   of tyTuple:
-    if retType != nil and retType.kind == tyLent:
-      result = true
-    else:
-      result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
-  else: result = false
+    result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
+  else:
+    result = false
+  # first parameter and return type is 'lent T'? --> use pass by pointer
+  if s.position == 0 and retType != nil and retType.kind == tyLent:
+    result = true
 
 proc fillResult(conf: ConfigRef; param: PNode) =
   fillLoc(param.sym.loc, locParam, param, ~"Result",
@@ -368,7 +367,7 @@ proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope =
   if result != nil: return
   result = getTypePre(m, typ, sig)
   if result != nil: return
-  let concrete = typ.skipTypes(abstractInst + {tyOpt})
+  let concrete = typ.skipTypes(abstractInst)
   case concrete.kind
   of tySequence, tyTuple, tyObject:
     result = getTypeName(m, typ, sig)

compiler/cgen.nim

@@ -2067,7 +2067,8 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
       let disp = generateMethodDispatchers(graph)
       for x in disp: genProcAux(m, x.sym)
 
-  m.g.modulesClosed.add m
+  let mm = m
+  m.g.modulesClosed.add mm
 
 proc genForwardedProcs(g: BModuleList) =
   # Forward declared proc:s lack bodies when first encountered, so they're given

compiler/closureiters.nim

@@ -129,7 +129,7 @@
 #   break :stateLoop
 
 import
-  ast, msgs, idents,
+  ast, astalgo, msgs, idents,
   renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
 
 type
@@ -388,6 +388,10 @@ proc hasYieldsInExpressions(n: PNode): bool =
           return true
     else:
       result = n.hasYields
+  of nkCast:
+    for i in 1..<n.len:
+      if n[i].hasYieldsInExpressions:
+        return true
   else:
     for c in n:
       if c.hasYieldsInExpressions:
@@ -686,7 +690,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
   of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv,
       nkDerefExpr, nkHiddenDeref:
     var ns = false
-    for i in 0..<n.len:
+    for i in ord(n.kind == nkCast)..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:

compiler/commands.nim

@@ -98,7 +98,7 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
                                  CPU[conf.target.hostCPU].name, CompileDate]),
                {msgStdout})
 
-    const gitHash = gorge("git log -n 1 --format=%H").strip
+    const gitHash {.strdefine.} = gorge("git log -n 1 --format=%H").strip
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
@@ -440,7 +440,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
   of "docroot":
-    conf.docRoot = if arg.len == 0: "@default" else: arg
+    conf.docRoot = if arg.len == 0: docRootDefault else: arg
   of "backend", "b":
     let backend = parseEnum(arg.normalize, TBackend.default)
     if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
@@ -485,7 +485,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "forcebuild", "f":
     processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
   of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
   of "gc":
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
@@ -913,8 +913,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
 
-template gCmdLineInfo*(): untyped = newLineInfo(commandLineIdx, 1, 1)
-
 proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   var cmd, arg: string
   splitSwitch(config, switch, cmd, arg, pass, gCmdLineInfo)

compiler/condsyms.nim

@@ -118,3 +118,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasInvariant")
   defineSymbol("nimHasStacktraceMsgs")
   defineSymbol("nimDoesntTrackDefects")
+  defineSymbol("nimHasLentIterators")