Introduce an AST for preprocessing + postprocessing

This reduces the complexity in the adapters, reduces
the caching cost per entry and speeds up deserialization.

Author: José Valim

lib/ecto/adapter.ex

@@ -7,8 +7,7 @@ defmodule Ecto.Adapter do
   @type t :: module
 
   @typedoc "Ecto.Query metadata fields (stored in cache)"
-  @type query_meta :: %{prefix: binary | nil, sources: tuple, assocs: term,
-                        preloads: term, select: term, fields: [term]}
+  @type query_meta :: %{prefix: binary | nil, sources: tuple, preloads: term, select: map}
 
   @typedoc "Ecto.Schema metadata fields"
   @type schema_meta :: %{source: source, schema: atom, context: term, autogenerate_id: {atom, :id | :binary_id}}
@@ -20,7 +19,7 @@ defmodule Ecto.Adapter do
   @type returning :: [atom]
   @type prepared :: term
   @type cached :: term
-  @type process :: (field :: Macro.t, value :: term, context :: term -> term)
+  @type process :: (term -> term)
   @type autogenerate_id :: {field :: atom, type :: :id | :binary_id, value :: term} | nil
   @type on_conflict :: {:raise, list(), []} |
                        {:nothing, list(), [atom]} |

lib/ecto/adapters/mysql/connection.ex

@@ -338,13 +338,10 @@ if Code.ensure_loaded?(Mariaex) do
       [name, ?. | quote_name(field)]
     end
 
-    defp expr({:&, _, [idx, fields, _counter]}, sources, query) do
-      {source, name, schema} = elem(sources, idx)
-      if is_nil(schema) and is_nil(fields) do
-        error!(query, "MySQL does not support selecting all fields from #{source} without a schema. " <>
-                      "Please specify a schema or specify exactly which fields you want to select")
-      end
-      intersperse_map(fields, ", ", &[name, ?. | quote_name(&1)])
+    defp expr({:&, _, [idx]}, sources, query) do
+      {source, _name, _schema} = elem(sources, idx)
+      error!(query, "MySQL does not support selecting all fields from #{source} without a schema. " <>
+                    "Please specify a schema or specify exactly which fields you want to select")
     end
 
     defp expr({:in, _, [_left, []]}, _sources, _query) do
@@ -377,8 +374,8 @@ if Code.ensure_loaded?(Mariaex) do
       ["NOT (", expr(expr, sources, query), ?)]
     end
 
-    defp expr(%Ecto.SubQuery{query: query, fields: fields}, _sources, _query) do
-      query.select.fields |> put_in(fields) |> all()
+    defp expr(%Ecto.SubQuery{query: query}, _sources, _query) do
+      all(query)
     end
 
     defp expr({:fragment, _, [kw]}, _sources, query) when is_list(kw) or tuple_size(kw) == 3 do

lib/ecto/adapters/postgres/connection.ex

@@ -418,13 +418,10 @@ if Code.ensure_loaded?(Postgrex) do
       quote_qualified_name(field, sources, idx)
     end
 
-    defp expr({:&, _, [idx, fields, _counter]}, sources, query) do
-      {source, name, schema} = elem(sources, idx)
-      if is_nil(schema) and is_nil(fields) do
-        error!(query, "PostgreSQL does not support selecting all fields from #{source} without a schema. " <>
-                      "Please specify a schema or specify exactly which fields you want to select")
-      end
-      intersperse_map(fields, ", ", &[name, ?. | quote_name(&1)])
+    defp expr({:&, _, [idx]}, sources, query) do
+      {source, _name, _schema} = elem(sources, idx)
+      error!(query, "PostgreSQL does not support selecting all fields from #{source} without a schema. " <>
+                    "Please specify a schema or specify exactly which fields you want to select")
     end
 
     defp expr({:in, _, [_left, []]}, _sources, _query) do
@@ -452,8 +449,8 @@ if Code.ensure_loaded?(Postgrex) do
       ["NOT (", expr(expr, sources, query), ?)]
     end
 
-    defp expr(%Ecto.SubQuery{query: query, fields: fields}, _sources, _query) do
-      query.select.fields |> put_in(fields) |> all()
+    defp expr(%Ecto.SubQuery{query: query}, _sources, _query) do
+      all(query)
     end
 
     defp expr({:fragment, _, [kw]}, _sources, query) when is_list(kw) or tuple_size(kw) == 3 do

lib/ecto/adapters/sql.ex

@@ -388,32 +388,15 @@ defmodule Ecto.Adapters.SQL do
     do_execute(repo, meta, prepared, params, mapper, put_source(opts, meta))
   end
 
-  defp do_execute(repo, _meta, {:cache, update, {id, prepared}}, params, nil, opts) do
-    execute_and_cache(repo, id, update, prepared, params, nil, opts)
-  end
-
-  defp do_execute(repo, %{fields: fields, sources: sources}, {:cache, update, {id, prepared}}, params, process, opts) do
-    mapper = &process_row(&1, process, fields, sources)
+  defp do_execute(repo, _meta, {:cache, update, {id, prepared}}, params, mapper, opts) do
     execute_and_cache(repo, id, update, prepared, params, mapper, opts)
   end
 
-  defp do_execute(repo, _meta, {:cached, reset, {id, cached}}, params, nil, opts) do
-    execute_or_reset(repo, id, reset, cached, params, nil, opts)
-  end
-
-  defp do_execute(repo, %{fields: fields, sources: sources}, {:cached, reset, {id, cached}}, params, process, opts) do
-    mapper = &process_row(&1, process, fields, sources)
+  defp do_execute(repo, _meta, {:cached, reset, {id, cached}}, params, mapper, opts) do
     execute_or_reset(repo, id, reset, cached, params, mapper, opts)
   end
 
-  defp do_execute(repo, _meta, {:nocache, {_id, prepared}}, params, nil, opts) do
-    %{rows: rows, num_rows: num} =
-      sql_call!(repo, :execute, [prepared], params, nil, opts)
-    {num, rows}
-  end
-
-  defp do_execute(repo, %{fields: fields, sources: sources}, {:nocache, {_id, prepared}}, params, process, opts) do
-    mapper = &process_row(&1, process, fields, sources)
+  defp do_execute(repo, _meta, {:nocache, {_id, prepared}}, params, mapper, opts) do
     %{rows: rows, num_rows: num} =
       sql_call!(repo, :execute, [prepared], params, mapper, opts)
     {num, rows}
@@ -491,30 +474,15 @@ defmodule Ecto.Adapters.SQL do
     do_stream(repo, meta, prepared, params, mapper, put_source(opts, meta))
   end
 
-  def do_stream(repo, _meta, {:cache, _, {_, prepared}}, params, nil, opts) do
-    prepare_stream(repo, prepared, params, nil, opts)
-  end
-
-  def do_stream(repo, %{fields: fields, sources: sources}, {:cache, _, {_, prepared}}, params, process, opts) do
-    mapper = &process_row(&1, process, fields, sources)
+  def do_stream(repo, _meta, {:cache, _, {_, prepared}}, params, mapper, opts) do
     prepare_stream(repo, prepared, params, mapper, opts)
   end
 
-  def do_stream(repo, _, {:cached, _, {_, cached}}, params, nil, opts) do
-    prepare_stream(repo, String.Chars.to_string(cached), params, nil, opts)
-  end
-
-  def do_stream(repo, %{fields: fields, sources: sources}, {:cached, _, {_, cached}}, params, process, opts) do
-    mapper = &process_row(&1, process, fields, sources)
+  def do_stream(repo, _, {:cached, _, {_, cached}}, params, mapper, opts) do
     prepare_stream(repo, String.Chars.to_string(cached), params, mapper, opts)
   end
 
-  def do_stream(repo, _meta, {:nocache, {_id, prepared}}, params, nil, opts) do
-    prepare_stream(repo, prepared, params, nil, opts)
-  end
-
-  def do_stream(repo, %{fields: fields, sources: sources}, {:nocache, {_id, prepared}}, params, process, opts) do
-    mapper = &process_row(&1, process, fields, sources)
+  def do_stream(repo, _meta, {:nocache, {_id, prepared}}, params, mapper, opts) do
     prepare_stream(repo, prepared, params, mapper, opts)
   end
 
@@ -570,33 +538,6 @@ defmodule Ecto.Adapters.SQL do
     end
   end
 
-  defp process_row(row, process, fields, sources) do
-    num_sources = tuple_size(sources)
-    Enum.map_reduce(fields, row, fn
-      {:&, _, [_, _, counter]} = field, acc when num_sources == 1 ->
-        {val, rest} = Enum.split(acc, counter)
-        {process.(field, val, nil), rest}
-      {:&, _, [_, _, counter]} = field, acc ->
-        case split_and_not_nil(acc, counter, true, []) do
-          {nil, rest} -> {nil, rest}
-          {val, rest} -> {process.(field, val, nil), rest}
-        end
-      field, [h|t] ->
-        {process.(field, h, nil), t}
-    end) |> elem(0)
-  end
-
-  defp split_and_not_nil(rest, 0, true, _acc), do: {nil, rest}
-  defp split_and_not_nil(rest, 0, false, acc), do: {:lists.reverse(acc), rest}
-
-  defp split_and_not_nil([nil|t], count, all_nil?, acc) do
-    split_and_not_nil(t, count - 1, all_nil?, [nil|acc])
-  end
-
-  defp split_and_not_nil([h|t], count, _all_nil?, acc) do
-    split_and_not_nil(t, count - 1, false, [h|acc])
-  end
-
   ## Transactions
 
   @doc false

lib/ecto/query.ex

@@ -2,7 +2,7 @@ defmodule Ecto.SubQuery do
   @moduledoc """
   Stores subquery information.
   """
-  defstruct [:query, :params, :fields, :meta, :cache]
+  defstruct [:query, :params, :select, :cache]
 end
 
 defmodule Ecto.Query do