[JuliaLowering] get macro name in ctx.world; fix lowering iterator

Author: mlechu

JuliaLowering/src/eval.jl

@@ -44,7 +44,7 @@ function lower_init(ex::SyntaxTree, mod::Module, macro_world::UInt; expr_compat_
     LoweringIterator{typeof(graph)}(ctx, [(ex, false, 0)])
 end
 
-function lower_step(iter, push_mod=nothing)
+function lower_step(iter, world=Base.get_world_counter(), push_mod=nothing)
     if !isnothing(push_mod)
         push_layer!(iter.ctx, push_mod, false)
     end
@@ -71,7 +71,12 @@ function lower_step(iter, push_mod=nothing)
 
     k = kind(ex)
     if !(k in KSet"toplevel module")
-        ex = expand_forms_1(iter.ctx, ex)
+        ctx1 = let c = iter.ctx
+            MacroExpansionContext(
+                c.graph, c.bindings, c.scope_layers, c.scope_layer_stack,
+                c.expr_compat_mode, world)
+        end
+        ex = expand_forms_1(ctx1, ex)
         k = kind(ex)
     end
     if k == K"toplevel"
@@ -93,7 +98,7 @@ function lower_step(iter, push_mod=nothing)
         return Core.svec(:begin_module, newmod_name, std_defs, loc)
     else
         # Non macro expansion parts of lowering
-        ctx2, ex2 = expand_forms_2(iter.ctx, ex)
+        ctx2, ex2 = expand_forms_2(ctx1, ex)
         ctx3, ex3 = resolve_scopes(ctx2, ex2)
         ctx4, ex4 = convert_closures(ctx3, ex3)
         ctx5, ex5 = linearize_ir(ctx4, ex4)
@@ -378,7 +383,7 @@ function _to_lowered_expr(ex::SyntaxTree, stmt_offset::Int)
             ir
         end
     elseif k == K"Value"
-        ex.value
+        ex.value isa LineNumberNode ? QuoteNode(ex.value) : ex.value
     elseif k == K"goto"
         Core.GotoNode(ex[1].id + stmt_offset)
     elseif k == K"gotoifnot"
@@ -473,7 +478,7 @@ function _eval(mod, iter)
     new_mod = nothing
     result = nothing
     while true
-        thunk = lower_step(iter, new_mod)::Core.SimpleVector
+        thunk = lower_step(iter, Base.get_world_counter(), new_mod)::Core.SimpleVector
         new_mod = nothing
         type = thunk[1]::Symbol
         if type == :done
@@ -503,7 +508,7 @@ function _eval(mod, iter, new_mod=nothing)
     in_new_mod = !isnothing(new_mod)
     result = nothing
     while true
-        thunk = lower_step(iter, new_mod)::Core.SimpleVector
+        thunk = lower_step(iter, Base.get_world_counter(), new_mod)::Core.SimpleVector
         new_mod = nothing
         type = thunk[1]::Symbol
         if type == :done

JuliaLowering/src/macro_expansion.jl

@@ -158,25 +158,49 @@ function fixup_macro_name(ctx::MacroExpansionContext, ex::SyntaxTree)
     end
 end
 
-function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex::SyntaxTree)
-    # `ex1` might contain a nontrivial mix of scope layers so we can't just
-    # `eval()` it, as it's already been partially lowered by this point.
-    # Instead, we repeat the latter parts of `lower()` here.
-    ex1 = expand_forms_1(ctx, fixup_macro_name(ctx, ex))
-    ctx2, ex2 = expand_forms_2(ctx, ex1)
-    ctx3, ex3 = resolve_scopes(ctx2, ex2)
-    ctx4, ex4 = convert_closures(ctx3, ex3)
-    ctx5, ex5 = linearize_ir(ctx4, ex4)
+function _eval_dot(world::UInt, mod, ex::SyntaxTree)
+    if kind(ex) === K"."
+        mod = _eval_dot(world, mod, ex[1])
+        ex = ex[2]
+    end
+    kind(ex) in KSet"Identifier Symbol" && mod isa Module ?
+        Base.invoke_in_world(world, getproperty, mod, Symbol(ex.name_val)) :
+        nothing
+end
+
+# If macroexpand(ex[1]) is an identifier or dot-expression, we can simply grab
+# it from the scope layer's module in ctx.macro_world.  Otherwise, we need to
+# eval arbitrary code (which, TODO: does not use the correct world age, and it
+# isn't clear the language is meant to support this).
+function eval_macro_name(ctx::MacroExpansionContext, mctx::MacroContext, ex0::SyntaxTree)
     mod = current_layer(ctx).mod
-    expr_form = to_lowered_expr(ex5)
+    ex = fixup_macro_name(ctx, expand_forms_1(ctx, ex0))
     try
-        # Using Core.eval here fails when precompiling packages since we hit the
-        # user-facing error (in `jl_check_top_level_effect`) that warns that
-        # effects won't persist when eval-ing into a closed module.
-        # `jl_invoke_julia_macro` bypasses this by calling `jl_toplevel_eval` on
-        # the macro name.  This is fine assuming the first argument to the
-        # macrocall is effect-free.
-        ccall(:jl_toplevel_eval, Any, (Any, Any), mod, expr_form)
+        if kind(ex) === K"Value"
+            !(ex.value isa GlobalRef) ? ex.value :
+                Base.invoke_in_world(ctx.macro_world, getglobal,
+                                     ex.value.mod, ex.value.name)
+        elseif kind(ex) === K"Identifier"
+            layer = get(ex, :scope_layer, nothing)
+            if !isnothing(layer)
+                mod = ctx.scope_layers[layer].mod
+            end
+            Base.invoke_in_world(ctx.macro_world, getproperty,
+                                 mod, Symbol(ex.name_val))
+        elseif kind(ex) === K"." &&
+                (ed = _eval_dot(ctx.macro_world, mod, ex); !isnothing(ed))
+            ed
+        else
+            # `ex` might contain a nontrivial mix of scope layers so we can't
+            # just `eval()` it, as it's already been partially lowered by this
+            # point.  Instead, we repeat the latter parts of `lower()` here.
+            ctx2, ex2 = expand_forms_2(ctx, ex)
+            ctx3, ex3 = resolve_scopes(ctx2, ex2)
+            ctx4, ex4 = convert_closures(ctx3, ex3)
+            ctx5, ex5 = linearize_ir(ctx4, ex4)
+            expr_form = to_lowered_expr(ex5)
+            ccall(:jl_toplevel_eval, Any, (Any, Any), mod, expr_form)
+        end
     catch err
         throw(MacroExpansionError(mctx, ex, "Macro not found", :all, err))
     end

JuliaLowering/test/macros.jl

@@ -483,4 +483,26 @@ end
     @test_broken JuliaLowering.eval(test_mod, code) == ("outer x", "inner x")
 end
 
+# the lowering/eval iterator needs to expand in the correct world age (currently
+# the only way to hit this from user code is macros producing toplevel)
+@testset "macros defining macros" begin
+    @eval test_mod macro make_and_use_macro_toplevel()
+        Expr(:toplevel,
+             esc(:(macro from_toplevel_expansion()
+                   :(123)
+               end)),
+             esc(:(@from_toplevel_expansion())))
+    end
+
+    @test JuliaLowering.include_string(
+        test_mod, "@make_and_use_macro_toplevel()"; expr_compat_mode=true) === 123
+
+    if isdefined(test_mod, Symbol("@from_toplevel_expansion"))
+        Base.delete_binding(test_mod, Symbol("@from_toplevel_expansion"))
+    end
+
+    @test JuliaLowering.include_string(
+        test_mod, "@make_and_use_macro_toplevel()"; expr_compat_mode=false) === 123
+end
+
 end

JuliaLowering/test/macros_ir.jl

@@ -147,7 +147,7 @@ _never_exist = @m_not_exist 42
 #---------------------
 MacroExpansionError while expanding @m_not_exist in module Main.TestMod:
 _never_exist = @m_not_exist 42
-#              └──────────┘ ── Macro not found
+#               └─────────┘ ── Macro not found
 Caused by:
 UndefVarError: `@m_not_exist` not defined in `Main.TestMod`
 Suggestion: check for spelling errors or missing imports.