macroexpand: stop pre-running the hygiene pass

This prepares us to delete the separate hygiene pass, and make it part
of lowering resolve-scopes. This affects a few macros which expect to
run macroexpand then mangle the result more, and the result of those
macros is not already wrapped in `esc()`. The damage seems relatively
minor however, and generally a good improvement.

Author: vtjnash

base/docs/Docs.jl

@@ -292,6 +292,16 @@ function unblock(@nospecialize ex)
     return unblock(exs[1])
 end
 
+# peek through ex to figure out what kind of expression it may eventually act like
+# but ignoring scopes and line numbers
+function unescape(@nospecialize ex)
+    ex = unblock(ex)
+    while isexpr(ex, :escape) || isexpr(ex, :var"hygienic-scope")
+       ex = unblock(ex.args[1])
+    end
+    return ex
+end
+
 uncurly(@nospecialize ex) = isexpr(ex, :curly) ? ex.args[1] : ex
 
 namify(@nospecialize x) = astname(x, isexpr(x, :macro))::Union{Symbol,Expr,GlobalRef}
@@ -351,18 +361,19 @@ function metadata(__source__, __module__, expr, ismodule)
         fields = P[]
         last_docstr = nothing
         for each in (expr.args[3]::Expr).args
-            if isa(each, Symbol) || isexpr(each, :(::))
+            eachex = unescape(each)
+            if isa(eachex, Symbol) || isexpr(eachex, :(::))
                 # a field declaration
                 if last_docstr !== nothing
-                    push!(fields, P(namify(each::Union{Symbol,Expr}), last_docstr))
+                    push!(fields, P(namify(eachex::Union{Symbol,Expr}), last_docstr))
                     last_docstr = nothing
                 end
-            elseif isexpr(each, :function) || isexpr(each, :(=))
+            elseif isexpr(eachex, :function) || isexpr(eachex, :(=))
                 break
-            elseif isa(each, String) || isexpr(each, :string) || isexpr(each, :call) ||
-                (isexpr(each, :macrocall) && each.args[1] === Symbol("@doc_str"))
+            elseif isa(eachex, String) || isexpr(eachex, :string) || isexpr(eachex, :call) ||
+                (isexpr(eachex, :macrocall) && eachex.args[1] === Symbol("@doc_str"))
                 # forms that might be doc strings
-                last_docstr = each::Union{String,Expr}
+                last_docstr = each
             end
         end
         dict = :($(Dict{Symbol,Any})($([(:($(P)($(quot(f)), $d)))::Expr for (f, d) in fields]...)))

base/osutils.jl

@@ -16,7 +16,7 @@ macro static(ex)
         @label loop
         hd = ex.head
         if hd ∈ (:if, :elseif, :&&, :||)
-            cond = Core.eval(__module__, ex.args[1])
+            cond = Core.eval(__module__, ex.args[1])::Bool
             if xor(cond, hd === :||)
                 return esc(ex.args[2])
             elseif length(ex.args) == 3

base/util.jl

@@ -604,7 +604,7 @@ macro kwdef(expr)
         kwdefs = nothing
     end
     return quote
-        Base.@__doc__ $(esc(expr))
+        $(esc(:($Base.@__doc__ $expr)))
         $kwdefs
     end
 end

deps/checksums/Pkg-33b5e806ffbb4d61c0216b17fbca159b4a65bdd4.tar.gz/md5

@@ -0,0 +1 @@
+f3d5df21fefba26f5568f719fc237bac

deps/checksums/Pkg-33b5e806ffbb4d61c0216b17fbca159b4a65bdd4.tar.gz/sha512

@@ -0,0 +1 @@
+7ff5f580c1f66e5fff875601193918ec9dc40a9d5bfbb9de1900075d1a52cfbf3c702cd7293a0ddcca1f2f23f70e19d5824257256bc7b208cdc952022961d3a2

deps/checksums/Pkg-daf02a458ae6daa402a5dd6683c40d6910325c4e.tar.gz/md5

@@ -265,12 +265,6 @@ DocMeta.setdocmeta!(
     maybe_revise(:(using Base.BinaryPlatforms));
     recursive=true, warn=false,
 )
-DocMeta.setdocmeta!(
-    Pkg.LazilyInitializedFields,
-    :DocTestSetup,
-    maybe_revise(:(using Pkg.LazilyInitializedFields));
-    recursive=true, warn=false,
-)
 
 let r = r"buildroot=(.+)", i = findfirst(x -> occursin(r, x), ARGS)
     global const buildroot = i === nothing ? (@__DIR__) : first(match(r, ARGS[i]).captures)

deps/checksums/Pkg-daf02a458ae6daa402a5dd6683c40d6910325c4e.tar.gz/sha512

@@ -83,7 +83,7 @@ be passed instead!). For example:
 
 ```jldoctest; setup = :(using InteractiveUtils)
 julia> macroexpand(@__MODULE__, :(@edit println("")) )
-:(InteractiveUtils.edit(println, (Base.typesof)("")))
+:($(Expr(Symbol("hygienic-scope"), :(edit($(Expr(:escape, :println)), (Base.typesof)($(Expr(:escape, ""))))), InteractiveUtils)))
 ```
 
 The functions `Base.Meta.show_sexpr` and [`dump`](@ref) are used to display S-expr style views

doc/make.jl

@@ -535,20 +535,21 @@ this is an extremely useful tool for debugging macros):
 
 ```julia-repl sayhello2
 julia> ex = macroexpand(Main, :(@sayhello("human")) )
-:(Main.println("Hello, ", "human"))
+:($(Expr(Symbol("hygienic-scope"), :(println("Hello, ", "human")), Main)))
 
 julia> typeof(ex)
 Expr
 ```
 
-We can see that the `"human"` literal has been interpolated into the expression.
+We can see that the macro expression has been replaced by a `hygienic-scope` expression in
+its place and the `"human"` literal has been interpolated into the expression.
 
 There also exists a macro [`@macroexpand`](@ref) that is perhaps a bit more convenient than the `macroexpand` function:
 
 
 ```jldoctest sayhello2
 julia> @macroexpand @sayhello "human"
-:(println("Hello, ", "human"))
+:($(Expr(Symbol("hygienic-scope"), :(println("Hello, ", "human")), Main)))
 ```
 
 ### Hold up: why macros?
@@ -579,7 +580,7 @@ julia> typeof(ex)
 Expr
 
 julia> ex
-:(println("I execute at runtime. The argument is: ", $(Expr(:copyast, :($(QuoteNode(:((1, 2, 3)))))))))
+:($(Expr(Symbol("hygienic-scope"), :(println("I execute at runtime. The argument is: ", $(Expr(:copyast, :($(QuoteNode(:((1, 2, 3))))))))), Main)))
 
 julia> eval(ex)
 I execute at runtime. The argument is: (1, 2, 3)
@@ -719,18 +720,18 @@ of a macro expansion with the aptly named [`@macroexpand`](@ref) macro:
 
 ```julia-repl assert2
 julia> @macroexpand @assert a == b
-:(if Main.a == Main.b
-        Main.nothing
-    else
-        Main.throw(Main.AssertionError("a == b"))
-    end)
+:($(Expr(Symbol("hygienic-scope"), :(if $(Expr(:escape, :(a == b)))
+      nothing
+  else
+      throw(AssertionError("a == b"))
+  end), Base)))
 
 julia> @macroexpand @assert a==b "a should equal b!"
-:(if Main.a == Main.b
-        Main.nothing
-    else
-        Main.throw(Main.AssertionError("a should equal b!"))
-    end)
+:($(Expr(Symbol("hygienic-scope"), :(if $(Expr(:escape, :(a == b)))
+      nothing
+  else
+      throw(AssertionError("a should equal b!"))
+  end), Base)))
 ```
 
 There is yet another case that the actual `@assert` macro handles: what if, in addition to printing
@@ -768,15 +769,22 @@ of expressions inside the macro body.
 ### Hygiene
 
 An issue that arises in more complex macros is that of [hygiene](https://en.wikipedia.org/wiki/Hygienic_macro).
-In short, macros must ensure that the variables they introduce in their returned expressions do
-not accidentally clash with existing variables in the surrounding code they expand into. Conversely,
-the expressions that are passed into a macro as arguments are often *expected* to evaluate in
-the context of the surrounding code, interacting with and modifying the existing variables. Another
-concern arises from the fact that a macro may be called in a different module from where it was
-defined. In this case we need to ensure that all global variables are resolved to the correct
-module. Julia already has a major advantage over languages with textual macro expansion (like
-C) in that it only needs to consider the returned expression. All the other variables (such as
-`msg` in `@assert` above) follow the [normal scoping block behavior](@ref scope-of-variables).
+We saw this a bit earlier with the hygienic-scope expression in the result of macroexpand
+that demarcated where our expression came from as a result of interpolation of the expanded
+macro into the rest of the expression.
+
+This already gives Julia a major advantage over languages with textual macro expansion (like
+C) in that it only needs to consider the returned expression, and it cannot change the
+parsing of the expression surrounding it. Thus, all the other variables (such as `msg` in
+`@assert` above) follow the [normal scoping block behavior](@ref scope-of-variables).
+
+Hygiene markers exists because macros must ensure that the variables they introduce in their
+returned expressions do not accidentally clash with existing variables in the surrounding
+code they expand into. Conversely, the expressions that are passed into a macro as arguments
+are often *expected* to evaluate in the context of the surrounding code, interacting with
+and modifying the existing variables. Another concern arises from the fact that a macro may
+be called in a different module from where it was defined. In this case we need to ensure
+that all global variables are resolved to the correct module.
 
 To demonstrate these issues, let us consider writing a `@time` macro that takes an expression
 as its argument, records the time, evaluates the expression, records the time again, prints the

doc/src/devdocs/reflection.md

@@ -998,6 +998,44 @@ int jl_has_meta(jl_array_t *body, jl_sym_t *sym) JL_NOTSAFEPOINT
     return 0;
 }
 
+// Utility function to return whether `e` is any of the special AST types or
+// will always evaluate to itself exactly unchanged. This corresponds to
+// `is_self_quoting` in Core.Compiler utilities.
+int jl_is_ast_node(jl_value_t *e) JL_NOTSAFEPOINT
+{
+    return jl_is_newvarnode(e)
+        || jl_is_code_info(e)
+        || jl_is_linenode(e)
+        || jl_is_gotonode(e)
+        || jl_is_gotoifnot(e)
+        || jl_is_returnnode(e)
+        || jl_is_ssavalue(e)
+        || jl_is_slotnumber(e)
+        || jl_is_argument(e)
+        || jl_is_quotenode(e)
+        || jl_is_globalref(e)
+        || jl_is_symbol(e)
+        || jl_is_pinode(e)
+        || jl_is_phinode(e)
+        || jl_is_phicnode(e)
+        || jl_is_upsilonnode(e)
+        || jl_is_expr(e);
+}
+
+// any AST, except those that cannot contain symbols
+// and have no side effects
+int need_esc_node(jl_value_t *e) JL_NOTSAFEPOINT
+{
+    if (jl_is_linenode(e)
+        || jl_is_ssavalue(e)
+        || jl_is_slotnumber(e)
+        || jl_is_argument(e)
+        || jl_is_quotenode(e))
+        return 0;
+    // note: jl_is_globalref(e) is not included here, since we care a little about about having a line number for it
+    return jl_is_ast_node(e);
+}
+
 static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule, jl_module_t **ctx, size_t world, int throw_load_error)
 {
     jl_task_t *ct = jl_current_task;
@@ -1080,7 +1118,11 @@ static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, str
         newctx.parent = macroctx;
         jl_value_t *a = jl_exprarg(e, 0);
         jl_value_t *a2 = jl_expand_macros(a, inmodule, &newctx, onelevel, world, throw_load_error);
-        if (a != a2)
+        if (jl_is_expr(a2) && ((jl_expr_t*)a2)->head == jl_escape_sym)
+            expr = jl_exprarg(a2, 0);
+        else if (!need_esc_node(a2))
+            expr = a2;
+        else if (a != a2)
             jl_array_ptr_set(e->args, 0, a2);
         return expr;
     }
@@ -1089,6 +1131,8 @@ static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, str
         newctx.m = macroctx ? macroctx->m : inmodule;
         newctx.parent = macroctx;
         jl_value_t *result = jl_invoke_julia_macro(e->args, inmodule, &newctx.m, world, throw_load_error);
+        if (!need_esc_node(result))
+            return result;
         jl_value_t *wrap = NULL;
         JL_GC_PUSH3(&result, &wrap, &newctx.m);
         // copy and wrap the result in `(hygienic-scope ,result ,newctx)
@@ -1099,10 +1143,13 @@ static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, str
         result = jl_copy_ast(result);
         if (!onelevel)
             result = jl_expand_macros(result, inmodule, wrap ? &newctx : macroctx, onelevel, world, throw_load_error);
-        if (wrap) {
+        if (wrap && need_esc_node(result)) {
             jl_exprargset(wrap, 0, result);
             jl_exprargset(wrap, 1, newctx.m);
-            result = wrap;
+            if (jl_is_expr(result) && ((jl_expr_t*)result)->head == jl_escape_sym)
+                result = jl_exprarg(result, 0);
+            else
+                result = wrap;
         }
         JL_GC_POP();
         return result;
@@ -1144,7 +1191,6 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand(jl_value_t *expr, jl_module_t *inmodule)
     JL_GC_PUSH1(&expr);
     expr = jl_copy_ast(expr);
     expr = jl_expand_macros(expr, inmodule, NULL, 0, jl_atomic_load_acquire(&jl_world_counter), 0);
-    expr = jl_call_scm_on_ast("jl-expand-macroscope", expr, inmodule);
     JL_GC_POP();
     return expr;
 }
@@ -1155,7 +1201,6 @@ JL_DLLEXPORT jl_value_t *jl_macroexpand1(jl_value_t *expr, jl_module_t *inmodule
     JL_GC_PUSH1(&expr);
     expr = jl_copy_ast(expr);
     expr = jl_expand_macros(expr, inmodule, NULL, 1, jl_atomic_load_acquire(&jl_world_counter), 0);
-    expr = jl_call_scm_on_ast("jl-expand-macroscope", expr, inmodule);
     JL_GC_POP();
     return expr;
 }

doc/src/manual/metaprogramming.md

@@ -173,10 +173,6 @@
 (define (jl-expand-to-thunk-stmt expr file line)
   (expand-to-thunk-stmt- expr file line))
 
-(define (jl-expand-macroscope expr)
-  (error-wrap (lambda ()
-                (julia-expand-macroscope expr))))
-
 ;; construct default definitions of `eval` for non-bare modules
 ;; called by jl_eval_module_expr
 (define (module-default-defs e)

src/ast.c

@@ -1,4 +1,4 @@
 PKG_BRANCH = master
-PKG_SHA1 = daf02a458ae6daa402a5dd6683c40d6910325c4e
+PKG_SHA1 = 33b5e806ffbb4d61c0216b17fbca159b4a65bdd4
 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git
 PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1

src/jlfrontend.scm

@@ -642,7 +642,7 @@ macro m1_11993()
 end
 
 macro m2_11993()
-    Symbol("@m1_11993")
+    esc(Symbol("@m1_11993"))
 end
 
 @doc "This should document @m1... since its the result of expansion" @m2_11993