open + preferred syms

compiler/ast.nim

@@ -518,6 +518,8 @@ type
     nfFirstWrite # this node is a first write
     nfHasComment # node has a comment
     nfSkipFieldChecking # node skips field visable checking
+    nfOpenSym # node is a captured sym but can be overriden by local symbols
+    nfPreferredSym # node is a preferred sym in a symchoice
 
   TNodeFlags* = set[TNodeFlag]
   TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 47)
@@ -1092,7 +1094,7 @@ const
                                       nfIsRef, nfIsPtr, nfPreventCg, nfLL,
                                       nfFromTemplate, nfDefaultRefsParam,
                                       nfExecuteOnReload, nfLastRead,
-                                      nfFirstWrite, nfSkipFieldChecking}
+                                      nfFirstWrite, nfSkipFieldChecking, nfOpenSym, nfPreferredSym}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2

compiler/lookups.nim

@@ -315,6 +315,8 @@ type
     m*: PSym
     mode*: TOverloadIterMode
     symChoiceIndex*: int
+    symChoiceLastPreferred: bool
+    fallback: PSym
     currentScope: PScope
     importIdx: int
     marked: IntSet
@@ -590,10 +592,19 @@ proc lookUp*(c: PContext, n: PNode): PSym =
     if result == nil: result = errorUndeclaredIdentifierHint(c, n, n.ident)
   of nkSym:
     result = n.sym
+    if false and nfOpenSym in n.flags:
+      let alt = searchInScopes(c, result.name, amb)
+      if alt != nil and alt != result and not amb and alt.owner == c.p.owner:
+        result = alt
   of nkAccQuoted:
     var ident = considerQuotedIdent(c, n)
     result = searchInScopes(c, ident, amb)
     if result == nil: result = errorUndeclaredIdentifierHint(c, n, ident)
+  of nkOpenSymChoice, nkClosedSymChoice:
+    if nfPreferredSym in n[0].flags:
+      result = n[0].sym
+    else:
+      result = nil
   else:
     internalError(c.config, n.info, "lookUp")
     return nil
@@ -639,6 +650,11 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
     c.isAmbiguous = amb
   of nkSym:
     result = n.sym
+    if false and nfOpenSym in n.flags:
+      var amb = false
+      let alt = searchInScopes(c, result.name, amb)
+      if alt != nil and alt != result and not amb and alt.owner == c.p.owner:
+        result = alt
   of nkDotExpr:
     result = nil
     var m = qualifiedLookUp(c, n[0], (flags * {checkUndeclared}) + {checkModule})
@@ -669,6 +685,11 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
         localError(c.config, n[1].info, "identifier expected, but got: " &
                    renderTree(n[1]))
         result = errorSym(c, n[1])
+  of nkOpenSymChoice, nkClosedSymChoice:
+    if nfPreferredSym in n[0].flags:
+      result = n[0].sym
+    else:
+      result = nil
   else:
     result = nil
   when false:
@@ -700,8 +721,25 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
           return nil
 
   of nkSym:
-    result = n.sym
-    o.mode = oimDone
+    if true or nfOpenSym notin n.flags:
+      result = n.sym
+      if nfPreferredSym in n.flags:
+        o.mode = oimSymChoiceLocalLookup
+        o.symChoiceLastPreferred = true
+        o.currentScope = c.currentScope
+        o.it.h = result.name.h
+        o.it.name = result.name
+        o.marked = initIntSet()
+        incl(o.marked, result.id)
+      else:
+        o.mode = oimDone
+    else:
+      result = initOverloadIter(o, c, newIdentNode(n.sym.name, n.info))
+      if result == nil:
+        result = n.sym
+        o.mode = oimDone
+      elif n.sym != result:
+        o.fallback = n.sym
   of nkDotExpr:
     result = nil
     o.mode = oimOtherModule
@@ -727,6 +765,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
     o.mode = oimSymChoice
     if n[0].kind == nkSym:
       result = n[0].sym
+      o.symChoiceLastPreferred = nfPreferredSym in n[0].flags
     else:
       o.mode = oimDone
       return nil
@@ -745,6 +784,11 @@ proc lastOverloadScope*(o: TOverloadIter): int =
              else: o.currentScope.depthLevel
   of oimSelfModule:  result = 1
   of oimOtherModule: result = 0
+  of oimSymChoice, oimSymChoiceLocalLookup:
+    if o.symChoiceLastPreferred:
+      result = 999
+    else:
+      result = -1
   else: result = -1
 
 proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym =
@@ -798,11 +842,15 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         result = nextOverloadIterImports(o, c, n)
     else:
       result = nil
+    if result == nil and o.fallback != nil and o.fallback.id notin o.marked:
+      result = o.fallback
+      o.fallback = nil
   of oimSelfModule:
     result = nextIdentIter(o.it, c.topLevelScope.symbols)
   of oimOtherModule:
     result = nextModuleIter(o.mit, c.graph)
   of oimSymChoice:
+    o.symChoiceLastPreferred = false
     if o.symChoiceIndex < n.len:
       result = n[o.symChoiceIndex].sym
       incl(o.marked, result.id)
@@ -827,13 +875,15 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
     else:
       result = nil
   of oimSymChoiceLocalLookup:
+    o.symChoiceLastPreferred = false
     if o.currentScope != nil:
       result = nextIdentExcluding(o.it, o.currentScope.symbols, o.marked)
       while result == nil:
         o.currentScope = o.currentScope.parent
         if o.currentScope != nil:
+          let name = if n.kind == nkSym: n.sym.name else: n[0].sym.name
           result = firstIdentExcluding(o.it, o.currentScope.symbols,
-                                      n[0].sym.name, o.marked)
+                                       name, o.marked)
         else:
           o.importIdx = 0
           result = symChoiceExtension(o, c, n)

compiler/semexprs.nim

@@ -101,7 +101,8 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType
   elif result.typ.kind == tyError:
     # associates the type error to the current owner
     result.typ = errorType(c)
-  elif efTypeAllowed in flags and result.typ.kind == tyProc and
+  elif {efTypeAllowed, efOperand} * flags != {} and
+      result.typ.kind == tyProc and
       hasUnresolvedParams(result, {}):
     # mirrored with semOperand but only on efTypeAllowed
     let owner = result.typ.owner
@@ -138,16 +139,11 @@ proc semSymChoice(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: P
   if isSymChoice(result) and efAllowSymChoice notin flags:
     # some contexts might want sym choices preserved for later disambiguation
     # in general though they are ambiguous
-    let first = n[0].sym
-    var foundSym: PSym = nil
-    if first.kind == skEnumField and
-        not isAmbiguous(c, first.name, {skEnumField}, foundSym) and
-        foundSym == first:
-      # choose the first resolved enum field, i.e. the latest in scope
-      # to mirror behavior before overloadable enums
-      result = n[0]
+    let first = n[0]
+    if result.len == 1 or nfPreferredSym in first.flags:
+      result = first
     else:
-      var err = "ambiguous identifier '" & first.name.s &
+      var err = "ambiguous identifier '" & first.sym.name.s &
         "' -- use one of the following:\n"
       for child in n:
         let candidate = child.sym
@@ -1025,7 +1021,8 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags; expectedTy
   of skMacro: result = semMacroExpr(c, result, orig, callee, flags, expectedType)
   of skTemplate: result = semTemplateExpr(c, result, callee, flags, expectedType)
   else:
-    semFinishOperands(c, result)
+    if callee.magic notin {mArrGet, mArrPut, mNBindSym}:
+      semFinishOperands(c, result)
     activate(c, result)
     fixAbstractType(c, result)
     analyseIfAddressTakenInCall(c, result)
@@ -1543,8 +1540,7 @@ proc builtinFieldAccess(c: PContext; n: PNode; flags: var TExprFlags): PNode =
 
 proc dotTransformation(c: PContext, n: PNode): PNode =
   if isSymChoice(n[1]) or
-      # generics usually leave field names as symchoices, but not types
-      (n[1].kind == nkSym and n[1].sym.kind == skType):
+      (n[1].kind == nkSym and n[1].sym.kind in routineKinds + {skType}):
     result = newNodeI(nkDotCall, n.info)
     result.add n[1]
     result.add copyTree(n[0])
@@ -2954,13 +2950,17 @@ proc getNilType(c: PContext): PType =
     c.nilTypeCache = result
 
 proc enumFieldSymChoice(c: PContext, n: PNode, s: PSym): PNode =
-  var o: TOverloadIter
+  var o: TOverloadIter = default(TOverloadIter)
+  var firstPreferred = true
   var i = 0
   var a = initOverloadIter(o, c, n)
+  let firstScope = lastOverloadScope(o)
   while a != nil:
     if a.kind == skEnumField:
       inc(i)
-      if i > 1: break
+      if i > 1:
+        firstPreferred = firstScope > lastOverloadScope(o)
+        break
     a = nextOverloadIter(o, c, n)
   let info = getCallLineInfo(n)
   if i <= 1:
@@ -2980,6 +2980,8 @@ proc enumFieldSymChoice(c: PContext, n: PNode, s: PSym): PNode =
         result.add newSymNode(a, info)
         onUse(info, a)
       a = nextOverloadIter(o, c, n)
+    if firstPreferred:
+      result[0].flags.incl nfPreferredSym
 
 proc semPragmaStmt(c: PContext; n: PNode) =
   if c.p.owner.kind == skModule:
@@ -3064,9 +3066,17 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
   of nkClosedSymChoice, nkOpenSymChoice:
     result = semSymChoice(c, result, flags, expectedType)
   of nkSym:
+    let s = n.sym
+    if nfOpenSym in n.flags:
+      let id = newIdentNode(s.name, n.info)
+      c.isAmbiguous = false
+      let s2 = qualifiedLookUp(c, id, {})
+      if s2 != nil and s2 != s and not c.isAmbiguous and s2.owner == c.p.owner:
+        result = semExpr(c, id, flags, expectedType)
+        return
     # because of the changed symbol binding, this does not mean that we
     # don't have to check the symbol for semantics here again!
-    result = semSym(c, n, n.sym, flags)
+    result = semSym(c, n, s, flags)
   of nkEmpty, nkNone, nkCommentStmt, nkType:
     discard
   of nkNilLit:

compiler/semgnrc.nim

@@ -69,6 +69,9 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
         result.transitionSonsKind(nkClosedSymChoice)
     else:
       result = symChoice(c, n, s, scOpen)
+      if withinMixin in flags and result.kind == nkSym:
+        result.flags.incl nfOpenSym
+        result.typ = nil
   case s.kind
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
@@ -96,6 +99,9 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
         result = n
     else:
       result = newSymNodeTypeDesc(s, c.idgen, n.info)
+      if withinMixin in flags:
+        result.flags.incl nfOpenSym
+        result.typ = nil
     onUse(n.info, s)
   of skParam:
     result = n
@@ -104,12 +110,18 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
     if (s.typ != nil) and
        (s.typ.flags * {tfGenericTypeParam, tfImplicitTypeParam} == {}):
       result = newSymNodeTypeDesc(s, c.idgen, n.info)
+      if withinMixin in flags:
+        result.flags.incl nfOpenSym
+        result.typ = nil
     else:
       result = n
     onUse(n.info, s)
   else:
     result = newSymNode(s, n.info)
     onUse(n.info, s)
+    if withinMixin in flags:
+      result.flags.incl nfOpenSym
+      result.typ = nil
 
 proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
             ctx: var GenericCtx): PNode =
@@ -148,6 +160,7 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
 
   var s = qualifiedLookUp(c, n, luf)
   if s != nil:
+    isMacro = s.kind in {skTemplate, skMacro}
     result = semGenericStmtSymbol(c, n, s, ctx, flags)
   else:
     n[0] = semGenericStmt(c, n[0], flags, ctx)
@@ -212,7 +225,16 @@ proc semGenericStmt(c: PContext, n: PNode,
     var dummy: bool
     result = fuzzyLookup(c, n, flags, ctx, dummy)
   of nkSym:
-    let a = n.sym
+    var a = n.sym
+    if nfOpenSym in n.flags:
+      let id = newIdentNode(a.name, n.info)
+      c.isAmbiguous = false
+      let s2 = qualifiedLookUp(c, id, {})
+      if s2 != nil and s2 != a and not c.isAmbiguous and s2.owner == c.p.owner:
+        n.sym = s2
+        a = s2
+      if withinMixin notin flags:
+        n.flags.excl nfOpenSym
     let b = getGenSym(c, a)
     if b != a: n.sym = b
   of nkEmpty, succ(nkSym)..nkNilLit, nkComesFrom:

compiler/sempass2.nim

@@ -630,7 +630,7 @@ proc isTrival(caller: PNode): bool {.inline.} =
 proc trackOperandForIndirectCall(tracked: PEffects, n: PNode, formals: PType; argIndex: int; caller: PNode) =
   let a = skipConvCastAndClosure(n)
   let op = a.typ
-  let param = if formals != nil and argIndex < formals.len and formals.n != nil: formals.n[argIndex].sym else: nil
+  let param = if formals != nil and formals.n != nil and argIndex < formals.n.len: formals.n[argIndex].sym else: nil
   # assume indirect calls are taken here:
   if op != nil and op.kind == tyProc and n.skipConv.kind != nkNilLit and
       not isTrival(caller) and
@@ -1115,7 +1115,7 @@ proc track(tracked: PEffects, n: PNode) =
     if n.sym.typ != nil and tfHasAsgn in n.sym.typ.flags:
       tracked.owner.flags.incl sfInjectDestructors
       # bug #15038: ensure consistency
-      if not hasDestructor(n.typ) and sameType(n.typ, n.sym.typ): n.typ = n.sym.typ
+      if n.typ != nil and not hasDestructor(n.typ) and sameType(n.typ, n.sym.typ): n.typ = n.sym.typ
   of nkHiddenAddr, nkAddr:
     if n[0].kind == nkSym and isLocalSym(tracked, n[0].sym):
       useVarNoInitCheck(tracked, n[0], n[0].sym)

compiler/semtempl.nim

@@ -52,13 +52,17 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule;
                isField = false): PNode =
   var
     a: PSym
-    o: TOverloadIter
+    o: TOverloadIter = default(TOverloadIter)
+    firstPreferred = true
   var i = 0
   a = initOverloadIter(o, c, n)
+  let firstScope = lastOverloadScope(o)
   while a != nil:
     if a.kind != skModule:
       inc(i)
-      if i > 1: break
+      if i > 1:
+        firstPreferred = firstScope > lastOverloadScope(o)
+        break
     a = nextOverloadIter(o, c, n)
   let info = getCallLineInfo(n)
   if i <= 1 and r != scForceOpen:
@@ -67,8 +71,16 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule;
     # for instance 'nextTry' is both in tables.nim and astalgo.nim ...
     if not isField or sfGenSym notin s.flags:
       result = newSymNode(s, info)
-      markUsed(c, info, s)
-      onUse(info, s)
+      if r == scClosed or n.kind == nkDotExpr or
+          (s.magic != mNone and s.kind in routineKinds):
+        markUsed(c, info, s)
+        onUse(info, s)
+      else:
+        # could maybe instead generate a open symchoice with a preferred sym,
+        # which the logic for is in the top else branch
+        result.flags.incl nfPreferredSym
+        incl(s.flags, sfUsed)
+        markOwnerModuleAsUsed(c, s)
     else:
       result = n
   elif i == 0:
@@ -88,6 +100,8 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule;
         result.add newSymNode(a, info)
         onUse(info, a)
       a = nextOverloadIter(o, c, n)
+    if r != scForceOpen and firstPreferred:
+      result[0].flags.incl nfPreferredSym
 
 proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode =
   result = copyNode(n)