From cd35df536bd6841098c658eacee18bc9b15a5091 Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Thu, 5 May 2016 22:03:14 -0400
Subject: [PATCH 1/9] implement comprehensions as `collect` of a `Generator`

this removes `static_typeof` and `type_goto`

fixes #7258
---
 base/abstractarray.jl       |   4 +-
 base/arraymath.jl           |   4 +-
 base/inference.jl           | 111 ++------------------------
 base/range.jl               |   2 +-
 base/sparse/sparsematrix.jl |  20 ++---
 src/alloc.c                 |   3 +-
 src/codegen.cpp             |  14 +---
 src/interpreter.c           |   5 +-
 src/jltypes.c               |   2 -
 src/julia-syntax.scm        | 152 +++---------------------------------
 src/julia.h                 |   4 +-
 src/toplevel.c              |   2 -
 test/arrayops.jl            |   2 +-
 test/dict.jl                |   2 +-
 14 files changed, 38 insertions(+), 289 deletions(-)

diff --git a/base/abstractarray.jl b/base/abstractarray.jl
index b8feeffa4d745..92874648302e1 100644
--- a/base/abstractarray.jl
+++ b/base/abstractarray.jl
@@ -804,8 +804,8 @@ typed_hcat{T}(::Type{T}) = Array{T}(0)
 ## cat: special cases
 vcat{T}(X::T...)         = T[ X[i] for i=1:length(X) ]
 vcat{T<:Number}(X::T...) = T[ X[i] for i=1:length(X) ]
-hcat{T}(X::T...)         = T[ X[j] for i=1, j=1:length(X) ]
-hcat{T<:Number}(X::T...) = T[ X[j] for i=1, j=1:length(X) ]
+hcat{T}(X::T...)         = T[ X[j] for i=1:1, j=1:length(X) ]
+hcat{T<:Number}(X::T...) = T[ X[j] for i=1:1, j=1:length(X) ]
 
 vcat(X::Number...) = hvcat_fill(Array{promote_typeof(X...)}(length(X)), X)
 hcat(X::Number...) = hvcat_fill(Array{promote_typeof(X...)}(1,length(X)), X)
diff --git a/base/arraymath.jl b/base/arraymath.jl
index c89ac3c2d6a30..53a73c3b5ef66 100644
--- a/base/arraymath.jl
+++ b/base/arraymath.jl
@@ -322,8 +322,8 @@ function ctranspose(A::AbstractMatrix)
 end
 ctranspose{T<:Real}(A::AbstractVecOrMat{T}) = transpose(A)
 
-transpose(x::AbstractVector) = [ transpose(v) for i=1, v in x ]
-ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=1, v in x ] #Fixme comprehension
+transpose(x::AbstractVector) = [ transpose(v) for i=1:1, v in x ]
+ctranspose{T}(x::AbstractVector{T}) = T[ ctranspose(v) for i=1:1, v in x ]
 
 _cumsum_type{T<:Number}(v::AbstractArray{T}) = typeof(+zero(T))
 _cumsum_type(v) = typeof(v[1]+v[1])
diff --git a/base/inference.jl b/base/inference.jl
index 7747386288874..e270b44a7e2ee 100644
--- a/base/inference.jl
+++ b/base/inference.jl
@@ -43,10 +43,8 @@ end
 type InferenceState
     sp::SimpleVector     # static parameters
     label_counter::Int   # index of the current highest label for this function
-    fedbackvars::Dict{SSAValue, Bool}
     mod::Module
     currpc::LineNum
-    static_typeof::Bool
 
     # info on the state of inference and the linfo
     linfo::LambdaInfo
@@ -71,7 +69,6 @@ type InferenceState
     backedges::Vector{Tuple{InferenceState, Vector{LineNum}}}
     # iteration fixed-point detection
     fixedpoint::Bool
-    typegotoredo::Bool
     inworkq::Bool
     # optimization
     optimize::Bool
@@ -158,13 +155,13 @@ type InferenceState
 
         inmodule = isdefined(linfo, :def) ? linfo.def.module : current_module() # toplevel thunks are inferred in the current module
         frame = new(
-            sp, nl, Dict{SSAValue, Bool}(), inmodule, 0, false,
+            sp, nl, inmodule, 0,
             linfo, la, s, Union{}, W, n,
             cur_hand, handler_at, n_handlers,
             ssavalue_uses, ssavalue_init,
             ObjectIdDict(), #Dict{InferenceState, Vector{LineNum}}(),
             Vector{Tuple{InferenceState, Vector{LineNum}}}(),
-            false, false, false, optimize, inlining, needtree, false)
+            false, false, optimize, inlining, needtree, false)
         push!(active, frame)
         nactive[] += 1
         return frame
@@ -1068,8 +1065,6 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState)
         return abstract_eval_constant(e)
     end
     e = e::Expr
-    # handle:
-    # call  null  new  &  static_typeof
     if is(e.head,:call)
         t = abstract_eval_call(e, vtypes, sv)
     elseif is(e.head,:null)
@@ -1103,42 +1098,6 @@ function abstract_eval(e::ANY, vtypes::VarTable, sv::InferenceState)
                 t = abstract_eval_constant(val)
             end
         end
-    elseif is(e.head,:static_typeof)
-        var = e.args[1]
-        t = widenconst(abstract_eval(var, vtypes, sv))
-        if isa(t,DataType) && typeseq(t,t.name.primary)
-            # remove unnecessary typevars
-            t = t.name.primary
-        end
-        if is(t,Bottom)
-            # if we haven't gotten fed-back type info yet, return Bottom. otherwise
-            # Bottom is the actual type of the variable, so return Type{Bottom}.
-            if get!(sv.fedbackvars, var, false)
-                t = Type{Bottom}
-            else
-                sv.static_typeof = true
-            end
-        elseif isleaftype(t)
-            t = Type{t}
-        elseif isleaftype(sv.linfo.specTypes)
-            if isa(t,TypeVar)
-                t = Type{t.ub}
-            else
-                t = Type{t}
-            end
-        else
-            # if there is any type uncertainty in the arguments, we are
-            # effectively predicting what static_typeof will say when
-            # the function is compiled with actual arguments. in that case
-            # abstract types yield Type{<:T} instead of Type{T}.
-            # this doesn't really model the situation perfectly, but
-            # "isleaftype(inference_stack.types)" should be good enough.
-            if isa(t,TypeVar) || isvarargtype(t)
-                t = Type{t}
-            else
-                t = Type{TypeVar(:_,t)}
-            end
-        end
     elseif is(e.head,:method)
         t = (length(e.args) == 1) ? Any : Void
     elseif is(e.head,:copyast)
@@ -1666,7 +1625,6 @@ function typeinf_frame(frame)
     W = frame.ip
     s = frame.stmt_types
     n = frame.nstmts
-    @label restart_typeinf
     while !isempty(W)
         # make progress on the active ip set
         local pc::Int = first(W), pc´::Int
@@ -1674,15 +1632,12 @@ function typeinf_frame(frame)
             #print(pc,": ",s[pc],"\n")
             delete!(W, pc)
             frame.currpc = pc
-            frame.static_typeof = false
             frame.cur_hand = frame.handler_at[pc]
             stmt = frame.linfo.code[pc]
             changes = abstract_interpret(stmt, s[pc]::Array{Any,1}, frame)
             if changes === ()
-                # if there was a Expr(:static_typeof) on this line,
-                # need to continue to the next pc even though the return type was Bottom
-                # otherwise, this line threw an error and there is no need to continue
-                frame.static_typeof || break
+                # this line threw an error and there is no need to continue
+                break
                 changes = s[pc]
             end
             if frame.cur_hand !== ()
@@ -1732,26 +1687,6 @@ function typeinf_frame(frame)
                             s[l] = newstate
                         end
                     end
-                elseif is(hd, :type_goto)
-                    for i = 2:length(stmt.args)
-                        var = stmt.args[i]::SSAValue
-                        # Store types that need to be fed back via type_goto
-                        # in ssavalue_init. After finishing inference, if any
-                        # of these types changed, start over with the fed-back
-                        # types known from the beginning.
-                        # See issue #3821 (using !typeseq instead of !subtype),
-                        # and issue #7810.
-                        id = var.id+1
-                        vt = frame.linfo.ssavaluetypes[id]
-                        ot = frame.ssavalue_init[id]
-                        if ot===NF || !(vt⊑ot && ot⊑vt)
-                            frame.ssavalue_init[id] = vt
-                            if get(frame.fedbackvars, var, false)
-                                frame.typegotoredo = true
-                            end
-                        end
-                        frame.fedbackvars[var] = true
-                    end
                 elseif is(hd, :return)
                     pc´ = n + 1
                     rt = abstract_eval(stmt.args[1], s[pc], frame)
@@ -1821,39 +1756,6 @@ function typeinf_frame(frame)
     end
 
     if finished || frame.fixedpoint
-        if frame.typegotoredo
-            # if any type_gotos changed, clear state and restart.
-            frame.typegotoredo = false
-            for ll = 2:length(s)
-                s[ll] = ()
-            end
-            empty!(W)
-            push!(W, 1)
-            frame.cur_hand = ()
-            frame.handler_at = Any[ () for i=1:n ]
-            frame.n_handlers = 0
-            frame.linfo.ssavaluetypes[:] = frame.ssavalue_init
-            @goto restart_typeinf
-        else
-            # if a static_typeof was never reached,
-            # use Union{} as its real type and continue
-            # running type inference from its uses
-            # (one of which is the static_typeof)
-            # TODO: this restart should happen just before calling finish()
-            for (fbvar, seen) in frame.fedbackvars
-                if !seen
-                    frame.fedbackvars[fbvar] = true
-                    id = (fbvar::SSAValue).id + 1
-                    for r in frame.ssavalue_uses[id]
-                        if !is(s[r], ()) # s[r] === () => unreached statement
-                            push!(W, r)
-                        end
-                    end
-                    @goto restart_typeinf
-                end
-            end
-        end
-
         if finished
             finish(frame)
         else # fixedpoint propagation
@@ -2056,7 +1958,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::InferenceState, undefs, pass)
 
     e = e::Expr
     head = e.head
-    if is(head,:static_typeof) || is(head,:line) || is(head,:const)
+    if is(head,:line) || is(head,:const)
         return e
     elseif is(head,:(=))
         e.args[2] = eval_annotate(e.args[2], vtypes, sv, undefs, pass)
@@ -2282,9 +2184,6 @@ function effect_free(e::ANY, sv, allow_volatile::Bool)
     elseif isa(e,Expr)
         e = e::Expr
         head = e.head
-        if head === :static_typeof
-            return true
-        end
         if head === :static_parameter || head === :meta || head === :line ||
             head === :inbounds || head === :boundscheck
             return true
diff --git a/base/range.jl b/base/range.jl
index a3badcf109535..335f2d89cb7d1 100644
--- a/base/range.jl
+++ b/base/range.jl
@@ -382,7 +382,7 @@ maximum(r::AbstractUnitRange) = isempty(r) ? throw(ArgumentError("range must be
 minimum(r::Range)  = isempty(r) ? throw(ArgumentError("range must be non-empty")) : min(first(r), last(r))
 maximum(r::Range)  = isempty(r) ? throw(ArgumentError("range must be non-empty")) : max(first(r), last(r))
 
-ctranspose(r::Range) = [x for _=1, x=r]
+ctranspose(r::Range) = [x for _=1:1, x=r]
 transpose(r::Range) = r'
 
 # Ranges are immutable
diff --git a/base/sparse/sparsematrix.jl b/base/sparse/sparsematrix.jl
index 6e18bf43b08ed..4ce6e0e0a4bc4 100644
--- a/base/sparse/sparsematrix.jl
+++ b/base/sparse/sparsematrix.jl
@@ -3072,8 +3072,8 @@ end
 
 function vcat(X::SparseMatrixCSC...)
     num = length(X)
-    mX = [ size(x, 1) for x in X ]
-    nX = [ size(x, 2) for x in X ]
+    mX = Int[ size(x, 1) for x in X ]
+    nX = Int[ size(x, 2) for x in X ]
     m = sum(mX)
     n = nX[1]
 
@@ -3090,7 +3090,7 @@ function vcat(X::SparseMatrixCSC...)
         Ti = promote_type(Ti, eltype(X[i].rowval))
     end
 
-    nnzX = [ nnz(x) for x in X ]
+    nnzX = Int[ nnz(x) for x in X ]
     nnz_res = sum(nnzX)
     colptr = Array{Ti}(n + 1)
     rowval = Array{Ti}(nnz_res)
@@ -3132,8 +3132,8 @@ end
 
 function hcat(X::SparseMatrixCSC...)
     num = length(X)
-    mX = [ size(x, 1) for x in X ]
-    nX = [ size(x, 2) for x in X ]
+    mX = Int[ size(x, 1) for x in X ]
+    nX = Int[ size(x, 2) for x in X ]
     m = mX[1]
     for i = 2 : num
         if mX[i] != m; throw(DimensionMismatch("")); end
@@ -3144,7 +3144,7 @@ function hcat(X::SparseMatrixCSC...)
     Ti = promote_type(map(x->eltype(x.rowval), X)...)
 
     colptr = Array{Ti}(n + 1)
-    nnzX = [ nnz(x) for x in X ]
+    nnzX = Int[ nnz(x) for x in X ]
     nnz_res = sum(nnzX)
     rowval = Array{Ti}(nnz_res)
     nzval = Array{Tv}(nnz_res)
@@ -3200,8 +3200,8 @@ Concatenate matrices block-diagonally. Currently only implemented for sparse mat
 """
 function blkdiag(X::SparseMatrixCSC...)
     num = length(X)
-    mX = [ size(x, 1) for x in X ]
-    nX = [ size(x, 2) for x in X ]
+    mX = Int[ size(x, 1) for x in X ]
+    nX = Int[ size(x, 2) for x in X ]
     m = sum(mX)
     n = sum(nX)
 
@@ -3209,7 +3209,7 @@ function blkdiag(X::SparseMatrixCSC...)
     Ti = promote_type(map(x->eltype(x.rowval), X)...)
 
     colptr = Array{Ti}(n + 1)
-    nnzX = [ nnz(x) for x in X ]
+    nnzX = Int[ nnz(x) for x in X ]
     nnz_res = sum(nnzX)
     rowval = Array{Ti}(nnz_res)
     nzval = Array{Tv}(nnz_res)
@@ -3450,7 +3450,7 @@ function trace{Tv}(A::SparseMatrixCSC{Tv})
     s
 end
 
-diag(A::SparseMatrixCSC) = [d for d in SpDiagIterator(A)]
+diag{Tv}(A::SparseMatrixCSC{Tv}) = Tv[d for d in SpDiagIterator(A)]
 
 function diagm{Tv,Ti}(v::SparseMatrixCSC{Tv,Ti})
     if (size(v,1) != 1 && size(v,2) != 1)
diff --git a/src/alloc.c b/src/alloc.c
index 545d968512aea..cded29bb6e57f 100644
--- a/src/alloc.c
+++ b/src/alloc.c
@@ -88,13 +88,12 @@ jl_sym_t *body_sym;
 jl_sym_t *method_sym;  jl_sym_t *core_sym;
 jl_sym_t *enter_sym;   jl_sym_t *leave_sym;
 jl_sym_t *exc_sym;     jl_sym_t *error_sym;
-jl_sym_t *static_typeof_sym;
 jl_sym_t *globalref_sym;
 jl_sym_t *new_sym;     jl_sym_t *using_sym;
 jl_sym_t *const_sym;   jl_sym_t *thunk_sym;
 jl_sym_t *anonymous_sym;  jl_sym_t *underscore_sym;
 jl_sym_t *abstracttype_sym; jl_sym_t *bitstype_sym;
-jl_sym_t *compositetype_sym; jl_sym_t *type_goto_sym;
+jl_sym_t *compositetype_sym;
 jl_sym_t *global_sym; jl_sym_t *list_sym;
 jl_sym_t *dot_sym;    jl_sym_t *newvar_sym;
 jl_sym_t *boundscheck_sym; jl_sym_t *inbounds_sym;
diff --git a/src/codegen.cpp b/src/codegen.cpp
index a8ff77a17ae83..589a7eb863c48 100644
--- a/src/codegen.cpp
+++ b/src/codegen.cpp
@@ -3110,7 +3110,7 @@ static void emit_stmtpos(jl_value_t *expr, jl_codectx_t *ctx)
     if (jl_is_expr(expr)) {
         jl_sym_t *head = ((jl_expr_t*)expr)->head;
         // some expression types are metadata and can be ignored in statement position
-        if (head == line_sym || head == type_goto_sym || head == meta_sym)
+        if (head == line_sym || head == meta_sym)
             return;
         // fall-through
     }
@@ -3284,18 +3284,6 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx)
                                literal_pointer_val(bnd));
         }
     }
-    else if (head == static_typeof_sym) {
-        jl_value_t *extype = expr_type((jl_value_t*)ex, ctx);
-        if (jl_is_type_type(extype)) {
-            extype = jl_tparam0(extype);
-            if (jl_is_typevar(extype))
-                extype = ((jl_tvar_t*)extype)->ub;
-        }
-        else {
-            extype = (jl_value_t*)jl_any_type;
-        }
-        return mark_julia_const(extype);
-    }
     else if (head == new_sym) {
         jl_value_t *ty = expr_type(args[0], ctx);
         size_t nargs = jl_array_len(ex->args);
diff --git a/src/interpreter.c b/src/interpreter.c
index 32900f7981f05..c348349962b9d 100644
--- a/src/interpreter.c
+++ b/src/interpreter.c
@@ -223,9 +223,6 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s)
     else if (ex->head == copyast_sym) {
         return jl_copy_ast(eval(args[0], s));
     }
-    else if (ex->head == static_typeof_sym) {
-        return (jl_value_t*)jl_any_type;
-    }
     else if (ex->head == exc_sym) {
         return ptls->exception_in_transit;
     }
@@ -429,7 +426,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s)
         jl_throw(args[0]);
     }
     else if (ex->head == boundscheck_sym || ex->head == inbounds_sym || ex->head == fastmath_sym ||
-             ex->head == simdloop_sym || ex->head == meta_sym || ex->head == type_goto_sym) {
+             ex->head == simdloop_sym || ex->head == meta_sym) {
         return jl_nothing;
     }
     jl_errorf("unsupported or misplaced expression %s", jl_symbol_name(ex->head));
diff --git a/src/jltypes.c b/src/jltypes.c
index 2739cbe8e8e35..47e1feeed6d64 100644
--- a/src/jltypes.c
+++ b/src/jltypes.c
@@ -3993,7 +3993,6 @@ void jl_init_types(void)
     exc_sym = jl_symbol("the_exception");
     enter_sym = jl_symbol("enter");
     leave_sym = jl_symbol("leave");
-    static_typeof_sym = jl_symbol("static_typeof");
     new_sym = jl_symbol("new");
     const_sym = jl_symbol("const");
     global_sym = jl_symbol("global");
@@ -4004,7 +4003,6 @@ void jl_init_types(void)
     abstracttype_sym = jl_symbol("abstract_type");
     bitstype_sym = jl_symbol("bits_type");
     compositetype_sym = jl_symbol("composite_type");
-    type_goto_sym = jl_symbol("type_goto");
     toplevel_sym = jl_symbol("toplevel");
     dot_sym = jl_symbol(".");
     boundscheck_sym = jl_symbol("boundscheck");
diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm
index 168352c1e1f10..9640baef3e3ec 100644
--- a/src/julia-syntax.scm
+++ b/src/julia-syntax.scm
@@ -1981,78 +1981,29 @@
 
    'comprehension
    (lambda (e)
-     (expand-forms (lower-comprehension #f       (cadr e) (cddr e))))
+     (if (any (lambda (x) (eq? x ':)) (cddr e))
+         (error "comprehension syntax with `:` ranges has been removed"))
+     (expand-forms `(call (top collect) (generator ,(cadr e) ,@(cddr e)))))
 
    'typed_comprehension
    (lambda (e)
+     (if (any (lambda (x) (eq? x ':)) (cdddr e))
+         (error "comprehension syntax with `:` ranges has been removed"))
      (expand-forms (lower-comprehension (cadr e) (caddr e) (cdddr e))))
 
    'dict_comprehension
    (lambda (e)
      (syntax-deprecation #f "[a=>b for (a,b) in c]" "Dict(a=>b for (a,b) in c)")
-     (expand-forms (lower-dict-comprehension (cadr e) (cddr e))))
+     (expand-forms `(call (top Dict) (generator ,(cadr e) ,@(cddr e)))))
 
    'typed_dict_comprehension
-   (lambda (e)
-     (expand-forms (lower-typed-dict-comprehension (cadr e) (caddr e) (cdddr e))))))
-
-(define (lower-nd-comprehension atype expr ranges)
-  (let ((result      (make-ssavalue))
-        (ri          (gensy))
-        (oneresult   (gensy)))
-    ;; evaluate one expression to figure out type and size
-    ;; compute just one value by inserting a break inside loops
-    (define (evaluate-one ranges)
-      (if (null? ranges)
-          `(= ,oneresult ,expr)
-          (if (eq? (car ranges) `:)
-              (evaluate-one (cdr ranges))
-              `(for ,(car ranges)
-                    (block ,(evaluate-one (cdr ranges))
-                           (break)) ))))
-
-    ;; compute the dimensions of the result
-    (define (compute-dims ranges oneresult-dim)
-      (if (null? ranges)
-          (list)
-          (if (eq? (car ranges) `:)
-              (cons `(call (top size) ,oneresult ,oneresult-dim)
-                    (compute-dims (cdr ranges) (+ oneresult-dim 1)))
-              (cons `(call (top length) ,(caddr (car ranges)))
-                    (compute-dims (cdr ranges) oneresult-dim)) )))
-
-    ;; construct loops to cycle over all dimensions of an n-d comprehension
-    (define (construct-loops ranges iters oneresult-dim)
-      (if (null? ranges)
-          (if (null? iters)
-              `(block (call (top setindex!) ,result ,expr ,ri)
-                      (= ,ri (call (top +) ,ri) 1))
-              `(block (call (top setindex!) ,result (ref ,expr ,@(reverse iters)) ,ri)
-                      (= ,ri (call (top +) ,ri 1))) )
-          (if (eq? (car ranges) `:)
-              (let ((i (make-ssavalue)))
-                `(for (= ,i (: 1 (call (top size) ,oneresult ,oneresult-dim)))
-                      ,(construct-loops (cdr ranges) (cons i iters) (+ oneresult-dim 1)) ))
-              `(for ,(car ranges)
-                    ,(construct-loops (cdr ranges) iters oneresult-dim) ))))
-
-    ;; Evaluate the comprehension
-    `(scope-block
-      (block
-       (local ,oneresult)
-       ,(evaluate-one ranges)
-       (= ,result (call (core Array) ,(if atype atype `(call (top eltype) ,oneresult))
-                        ,@(compute-dims ranges 1)))
-       (= ,ri 1)
-       ,(construct-loops (reverse ranges) (list) 1)
-       ,result ))))
+   (lambda (e) (expand-forms
+                `(call (call (core apply_type) (top Dict) ,@(cdr (cadr e)))
+                       (generator ,(caddr e) ,@(cdddr e)))))))
 
 (define (lower-comprehension atype expr ranges)
-  (if (any (lambda (x) (eq? x ':)) ranges)
-      (lower-nd-comprehension atype expr ranges)
   (let ((result    (make-ssavalue))
         (ri        (gensy))
-        (initlabl  (if atype #f (make-ssavalue)))
         (oneresult (make-ssavalue))
         (lengths   (map (lambda (x) (make-ssavalue)) ranges))
         (states    (map (lambda (x) (gensy)) ranges))
@@ -2063,7 +2014,6 @@
     (define (construct-loops ranges rv is states lengths)
       (if (null? ranges)
           `(block (= ,oneresult ,expr)
-                  ,@(if atype '() `((type_goto ,initlabl ,oneresult)))
                   (inbounds true)
                   (call (top setindex!) ,result ,oneresult ,ri)
                   (inbounds pop)
@@ -2089,79 +2039,10 @@
       ,.(map (lambda (v r) `(= ,v (call (top length) ,r))) lengths rv)
       (scope-block
        (block
-        ,@(if atype '() `((label ,initlabl)))
-        (= ,result (call (core Array)
-                         ,(if atype atype `(static_typeof ,oneresult))
-                         ,@lengths))
+        (= ,result (call (core Array) ,atype ,@lengths))
         (= ,ri 1)
         ,(construct-loops (reverse ranges) (reverse rv) is states (reverse lengths))
-        ,result))))))
-
-(define (lower-dict-comprehension expr ranges)
-  (let ((result   (make-ssavalue))
-        (initlabl (make-ssavalue))
-        (onekey   (make-ssavalue))
-        (oneval   (make-ssavalue))
-        (rv         (map (lambda (x) (make-ssavalue)) ranges)))
-
-    ;; construct loops to cycle over all dimensions of an n-d comprehension
-    (define (construct-loops ranges)
-      (if (null? ranges)
-          `(block (= ,onekey ,(cadr expr))
-                  (= ,oneval ,(caddr expr))
-                  (type_goto ,initlabl ,onekey ,oneval)
-                  (call (top setindex!) ,result ,oneval ,onekey))
-          `(for ,(car ranges)
-                (block
-                 ;; *** either this or force all for loop vars local
-                 ,.(map (lambda (r) `(local ,r))
-                        (lhs-vars (cadr (car ranges))))
-                 ,(construct-loops (cdr ranges))))))
-
-    ;; Evaluate the comprehension
-    (let ((loopranges
-           (map (lambda (r v) `(= ,(cadr r) ,v)) ranges rv)))
-      `(block
-        ,.(map (lambda (v r) `(= ,v ,(caddr r))) rv ranges)
-        (scope-block
-         (block
-          #;,@(map (lambda (r) `(local ,r))
-          (apply append (map (lambda (r) (lhs-vars (cadr r))) ranges)))
-          (label ,initlabl)
-          (= ,result (call (curly (top Dict)
-                                  (static_typeof ,onekey)
-                                  (static_typeof ,oneval))))
-          ,(construct-loops (reverse loopranges))
-          ,result))))))
-
-(define (lower-typed-dict-comprehension atypes expr ranges)
-  (if (not (and (length= atypes 3)
-                (eq? (car atypes) '=>)))
-      (error "invalid \"typed_dict_comprehension\" syntax")
-      (let ( (result (make-ssavalue))
-             (rs (map (lambda (x) (make-ssavalue)) ranges)) )
-
-        ;; construct loops to cycle over all dimensions of an n-d comprehension
-        (define (construct-loops ranges rs)
-          (if (null? ranges)
-              `(call (top setindex!) ,result ,(caddr expr) ,(cadr expr))
-              `(for (= ,(cadr (car ranges)) ,(car rs))
-                    (block
-                     ;; *** either this or force all for loop vars local
-                     ,.(map (lambda (r) `(local ,r))
-                            (lhs-vars (cadr (car ranges))))
-                     ,(construct-loops (cdr ranges) (cdr rs))))))
-
-        ;; Evaluate the comprehension
-        `(block
-          ,.(map make-assignment rs (map caddr ranges))
-          (= ,result (call (curly (top Dict) ,(cadr atypes) ,(caddr atypes))))
-          (scope-block
-           (block
-            #;,@(map (lambda (r) `(local ,r))
-            (apply append (map (lambda (r) (lhs-vars (cadr r))) ranges)))
-            ,(construct-loops (reverse ranges) (reverse rs))
-            ,result))))))
+        ,result)))))
 
 (define (lhs-vars e)
   (cond ((symbol? e) (list e))
@@ -3141,17 +3022,6 @@ f(x) = yt(x)
                      (cons (list code handler-level (cadr e)) handler-goto-fixups))
                '(null)))
 
-            ((type_goto)
-             (let ((m (get label-map (cadr e) #f)))
-               (if m
-                   (emit `(type_goto ,m ,@(cddr e)))
-                   (let ((l (make-label)))
-                     (put! label-map (cadr e) l)
-                     (emit `(type_goto ,l ,@(cddr e)))))))
-            ((static_typeof)
-             (assert (and value (not tail)))
-             e)
-
             ;; exception handlers are lowered using
             ;; (enter L) - push handler with catch block at label L
             ;; (leave n) - pop N exception handlers
diff --git a/src/julia.h b/src/julia.h
index ef280e0a896fc..808e30570c0c4 100644
--- a/src/julia.h
+++ b/src/julia.h
@@ -570,11 +570,11 @@ extern jl_sym_t *body_sym;
 extern jl_sym_t *method_sym;  extern jl_sym_t *slot_sym;
 extern jl_sym_t *enter_sym;   extern jl_sym_t *leave_sym;
 extern jl_sym_t *exc_sym;     extern jl_sym_t *new_sym;
-extern jl_sym_t *static_typeof_sym; extern jl_sym_t *compiler_temp_sym;
+extern jl_sym_t *compiler_temp_sym;
 extern jl_sym_t *const_sym;   extern jl_sym_t *thunk_sym;
 extern jl_sym_t *anonymous_sym;  extern jl_sym_t *underscore_sym;
 extern jl_sym_t *abstracttype_sym; extern jl_sym_t *bitstype_sym;
-extern jl_sym_t *compositetype_sym; extern jl_sym_t *type_goto_sym;
+extern jl_sym_t *compositetype_sym;
 extern jl_sym_t *global_sym; extern jl_sym_t *unused_sym;
 extern jl_sym_t *boundscheck_sym; extern jl_sym_t *inbounds_sym;
 extern jl_sym_t *copyast_sym; extern jl_sym_t *fastmath_sym;
diff --git a/src/toplevel.c b/src/toplevel.c
index 4ea1cb8972a8c..23db71a5e0fb6 100644
--- a/src/toplevel.c
+++ b/src/toplevel.c
@@ -269,8 +269,6 @@ int jl_has_intrinsics(jl_lambda_info_t *li, jl_value_t *v, jl_module_t *m)
     jl_expr_t *e = (jl_expr_t*)v;
     if (jl_array_len(e->args) == 0)
         return 0;
-    if (e->head == static_typeof_sym)
-        return 1;
     if (e->head == toplevel_sym || e->head == copyast_sym)
         return 0;
     jl_value_t *e0 = jl_exprarg(e, 0);
diff --git a/test/arrayops.jl b/test/arrayops.jl
index c7e0356e00b37..29377c403b5a2 100644
--- a/test/arrayops.jl
+++ b/test/arrayops.jl
@@ -983,7 +983,7 @@ X = [ i+2j for i=1:5, j=1:5 ]
 @test X[2,3] == 8
 @test X[4,5] == 14
 @test isequal(ones(2,3) * ones(2,3)', [3. 3.; 3. 3.])
-@test isequal([ [1,2] for i=1:2, : ], [1 2; 1 2])
+# @test isequal([ [1,2] for i=1:2, : ], [1 2; 1 2])
 # where element type is a Union. try to confuse type inference.
 foo32_64(x) = (x<2) ? Int32(x) : Int64(x)
 boo32_64() = [ foo32_64(i) for i=1:2 ]
diff --git a/test/dict.jl b/test/dict.jl
index c0cde55fcee63..45dcbbe998c8d 100644
--- a/test/dict.jl
+++ b/test/dict.jl
@@ -98,7 +98,7 @@ let
 end
 
 _d = Dict("a"=>0)
-@test isa([k for k in filter(x->length(x)==1, collect(keys(_d)))], Vector{Any})
+@test isa([k for k in filter(x->length(x)==1, collect(keys(_d)))], Vector{String})
 
 let
     d = Dict(((1, 2), (3, 4)))

From 66f12303ee142fd9fe1817e3a2df8869df684969 Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Fri, 13 May 2016 12:18:08 -0400
Subject: [PATCH 2/9] make `size` of a product iterator concatenate the sizes
 of its components

this removes an extra layer of wrapping with `IteratorND`

add some inline declarations for `Generator` iteration

remove unused `IteratorND` type
---
 base/generator.jl    |  5 +++--
 base/iterator.jl     | 34 ----------------------------------
 src/julia-syntax.scm |  2 +-
 test/functional.jl   | 15 ++++-----------
 4 files changed, 8 insertions(+), 48 deletions(-)

diff --git a/base/generator.jl b/base/generator.jl
index b3ef8523a706e..732aefca93903 100644
--- a/base/generator.jl
+++ b/base/generator.jl
@@ -17,9 +17,10 @@ Generator(f, I1, I2, Is...) = Generator(a->f(a...), zip(I1, I2, Is...))
 
 Generator{T,I}(::Type{T}, iter::I) = Generator{I,Type{T}}(T, iter)
 
-start(g::Generator) = start(g.iter)
-done(g::Generator, s) = done(g.iter, s)
+start(g::Generator) = (@_inline_meta; start(g.iter))
+done(g::Generator, s) = (@_inline_meta; done(g.iter, s))
 function next(g::Generator, s)
+    @_inline_meta
     v, s2 = next(g.iter, s)
     g.f(v), s2
 end
diff --git a/base/iterator.jl b/base/iterator.jl
index 843ecc7e4dc0a..56c97b7348618 100644
--- a/base/iterator.jl
+++ b/base/iterator.jl
@@ -421,40 +421,6 @@ prod_iteratorsize(::IsInfinite, b) = IsInfinite()
 prod_iteratorsize(a, b) = SizeUnknown()
 
 
-"""
-    IteratorND(iter, dims)
-
-Given an iterator `iter` and dimensions tuple `dims`, return an iterator that
-yields the same values as `iter`, but with the specified multi-dimensional shape.
-For example, this determines the shape of the array returned when `collect` is
-applied to this iterator.
-"""
-immutable IteratorND{I,N}
-    iter::I
-    dims::NTuple{N,Int}
-
-    function (::Type{IteratorND}){I,N}(iter::I, shape::NTuple{N,Integer})
-        li = length(iter)
-        if li != prod(shape)
-            throw(DimensionMismatch("dimensions $shape must be consistent with iterator length $li"))
-        end
-        new{I,N}(iter, shape)
-    end
-    (::Type{IteratorND}){I<:AbstractProdIterator}(p::I) = IteratorND(p, size(p))
-end
-
-start(i::IteratorND) = start(i.iter)
-done(i::IteratorND, s) = done(i.iter, s)
-next(i::IteratorND, s) = next(i.iter, s)
-
-size(i::IteratorND) = i.dims
-length(i::IteratorND) = prod(size(i))
-ndims{I,N}(::IteratorND{I,N}) = N
-iteratorsize{T<:IteratorND}(::Type{T}) = HasShape()
-
-eltype{I}(::IteratorND{I}) = eltype(I)
-iteratoreltype{I}(::Type{IteratorND{I}}) = iteratoreltype(I)
-
 # flatten an iterator of iterators
 
 immutable Flatten{I}
diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm
index 9640baef3e3ec..0018ecfd5c2cf 100644
--- a/src/julia-syntax.scm
+++ b/src/julia-syntax.scm
@@ -1977,7 +1977,7 @@
           `(call (top Generator) (-> ,argname (block ,@splat ,expr))
                  ,(if (length= ranges 1)
                       (car ranges)
-                      `(call (top IteratorND) (call (top product) ,@ranges))))))))
+                      `(call (top product) ,@ranges)))))))
 
    'comprehension
    (lambda (e)
diff --git a/test/functional.jl b/test/functional.jl
index 5042a81507bdc..e8f890f7ede7c 100644
--- a/test/functional.jl
+++ b/test/functional.jl
@@ -361,7 +361,10 @@ end
 @test Base.iteratoreltype(Base.product(take(1:2, 2)))               == Base.HasEltype()
 @test Base.iteratoreltype(Base.product([1 2; 3 4]))                 == Base.HasEltype()
 
-
+@test collect(Base.product(1:2,3:4)) == [(1,3) (1,4); (2,3) (2,4)]
+@test isempty(collect(Base.product(1:0,1:2)))
+@test length(Base.product(1:2,1:10,4:6)) == 60
+@test Base.iteratorsize(Base.product(1:2, countfrom(1))) == Base.IsInfinite()
 
 # flatten
 # -------
@@ -398,14 +401,6 @@ end
 @test collect((i+10j for i=1:2,j=3:4)) == [31 41; 32 42]
 @test collect((i+10j for i=1:2,j=3:4,k=1:1)) == reshape([31 41; 32 42], (2,2,1))
 
-let I = Base.IteratorND(1:27,(3,3,3))
-    @test collect(I) == reshape(1:27,(3,3,3))
-    @test size(I) == (3,3,3)
-    @test length(I) == 27
-    @test eltype(I) === Int
-    @test ndims(I) == 3
-end
-
 let A = collect(Base.Generator(x->2x, Real[1.5,2.5]))
     @test A == [3,5]
     @test isa(A,Vector{Float64})
@@ -415,8 +410,6 @@ let f(g) = (@test size(g.iter)==(2,3))
     f(i+j for i=1:2, j=3:5)
 end
 
-@test_throws DimensionMismatch Base.IteratorND(1:2, (2,3))
-
 @test collect(Base.Generator(+, [1,2], [10,20])) == [11,22]
 
 # generator ndims #16394

From 928bcde3b78951f4fd145423e547b2377d03cb21 Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Fri, 27 May 2016 18:30:52 -0400
Subject: [PATCH 3/9] use inferred type for empty comprehensions if it is a
 leaf type

this implements the "Steve Johnson compromise":
- non-empty comprehensions don't depend on inference at all
- in the empty case, we use either Union{} or the inferred type
- therefore there is no regression in type precision in cases where
  we can infer a leaf type

add Core.Inference.return_type

This computes a return type (at compile time if possible) in a way that handles
recursive dependencies in inference.
---
 base/array.jl     | 15 +++++++++++----
 base/inference.jl | 36 +++++++++++++++++++++++++++++++++---
 2 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/base/array.jl b/base/array.jl
index 4fc82d644a089..81e0cc2866fc5 100644
--- a/base/array.jl
+++ b/base/array.jl
@@ -244,16 +244,23 @@ function _collect(cont, itr, ::HasEltype, isz::SizeUnknown)
     return a
 end
 
-_default_eltype(itr::ANY) = Union{}
-_default_eltype{I,T}(::Generator{I,Type{T}}) = T
+if isdefined(Core, :Inference)
+    function _default_eltype(itrt::ANY)
+        rt = Core.Inference.return_type(first, Tuple{itrt})
+        return isleaftype(rt) ? rt : Union{}
+    end
+else
+    _default_eltype(itr::ANY) = Union{}
+end
+_default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T
 
 _collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) =
-    grow_to!(_similar_for(c, _default_eltype(itr), itr, isz), itr)
+    grow_to!(_similar_for(c, _default_eltype(typeof(itr)), itr, isz), itr)
 
 function _collect(c, itr, ::EltypeUnknown, isz::Union{HasLength,HasShape})
     st = start(itr)
     if done(itr,st)
-        return _similar_for(c, _default_eltype(itr), itr, isz)
+        return _similar_for(c, _default_eltype(typeof(itr)), itr, isz)
     end
     v1, st = next(itr, st)
     collect_to_with_first!(_similar_for(c, typeof(v1), itr, isz), v1, itr, st)
diff --git a/base/inference.jl b/base/inference.jl
index e270b44a7e2ee..aa76bba2becd1 100644
--- a/base/inference.jl
+++ b/base/inference.jl
@@ -923,13 +923,32 @@ function abstract_apply(af::ANY, fargs, aargtypes::Vector{Any}, vtypes::VarTable
     return abstract_call(af, (), Any[type_typeof(af), Vararg{Any}], vtypes, sv)
 end
 
-function pure_eval_call(f::ANY, argtypes::ANY, atype, sv)
+function pure_eval_call(f::ANY, argtypes::ANY, atype, vtypes, sv)
     for a in drop(argtypes,1)
         if !(isa(a,Const) || (isType(a) && !has_typevars(a.parameters[1])))
             return false
         end
     end
 
+    if f === return_type && length(argtypes) == 3
+        tt = argtypes[3]
+        if isType(tt)
+            af_argtype = tt.parameters[1]
+            if af_argtype <: Tuple && isa(af_argtype, DataType)
+                af = argtypes[2]
+                rt = abstract_call(isa(af,Const) ? af.val : af.parameters[1],
+                                   (), Any[argtypes[2], af_argtype.parameters...], vtypes, sv)
+                if isa(rt,Const)
+                    return Type{widenconst(rt)}
+                elseif isleaftype(rt) || isleaftype(af_argtype) || rt === Bottom
+                    return Type{rt}
+                else
+                    return Type{TypeVar(:R, rt)}
+                end
+            end
+        end
+    end
+
     meth = _methods_by_ftype(atype, 1)
     if meth === false || length(meth) != 1
         return false
@@ -1008,7 +1027,7 @@ function abstract_call(f::ANY, fargs, argtypes::Vector{Any}, vtypes::VarTable, s
     end
 
     atype = argtypes_to_type(argtypes)
-    t = pure_eval_call(f, argtypes, atype, sv)
+    t = pure_eval_call(f, argtypes, atype, vtypes, sv)
     t !== false && return t
 
     if istopfunction(tm, f, :promote_type) || istopfunction(tm, f, :typejoin)
@@ -2449,7 +2468,8 @@ function inlineable(f::ANY, ft::ANY, e::Expr, atypes::Vector{Any}, sv::Inference
     methsp = meth[2]
     method = meth[3]::Method
     # check whether call can be inlined to just a quoted constant value
-    if isa(f, widenconst(ft)) && !method.isstaged && method.lambda_template.pure && (isType(e.typ) || isa(e.typ,Const))
+    if isa(f, widenconst(ft)) && !method.isstaged && (method.lambda_template.pure || f === return_type) &&
+        (isType(e.typ) || isa(e.typ,Const))
         if isType(e.typ)
             if !has_typevars(e.typ.parameters[1])
                 return inline_as_constant(e.typ.parameters[1], argexprs, sv)
@@ -3564,6 +3584,16 @@ function reindex_labels!(linfo::LambdaInfo, sv::InferenceState)
     end
 end
 
+function return_type(f::ANY, t::ANY)
+    rt = Union{}
+    for m in _methods(f, t, -1)
+        _, ty, inferred = typeinf(m[3], m[1], m[2], false)
+        !inferred && return Any
+        rt = tmerge(rt, ty)
+        rt === Any && break
+    end
+    return rt
+end
 
 #### bootstrapping ####
 

From 354e9a9064c2c9595c2ad0ecc3168a03cc9adbd4 Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Tue, 21 Jun 2016 18:23:32 -0400
Subject: [PATCH 4/9] do some eta reduction (use `f` instead of `x->f(x)`) in
 Generator syntax

---
 src/julia-syntax.scm | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm
index 0018ecfd5c2cf..fa273a8fbc50a 100644
--- a/src/julia-syntax.scm
+++ b/src/julia-syntax.scm
@@ -1974,7 +1974,13 @@
                             `(,@(map (lambda (v) `(local ,v)) (lhs-vars `(tuple ,@vars)))
                               (= (tuple ,@vars) ,argname))))))
          (expand-forms
-          `(call (top Generator) (-> ,argname (block ,@splat ,expr))
+          `(call (top Generator)
+                 ,(if (and (null? splat)
+                           (length= expr 3) (eq? (car expr) 'call)
+                           (eq? (caddr expr) argname)
+                           (not (expr-contains-eq argname (cadr expr))))
+                      (cadr expr)  ;; eta reduce `x->f(x)` => `f`
+                      `(-> ,argname (block ,@splat ,expr)))
                  ,(if (length= ranges 1)
                       (car ranges)
                       `(call (top product) ,@ranges)))))))

From d09f30f646a81a9db2db6f1d6a24125f0a842def Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Wed, 22 Jun 2016 01:09:09 -0400
Subject: [PATCH 5/9] take advantage of shape-preserving comprehensions in
 vectorize macros

---
 base/operators.jl | 30 +++++++++---------------------
 1 file changed, 9 insertions(+), 21 deletions(-)

diff --git a/base/operators.jl b/base/operators.jl
index ae9ccd639d525..c6692bdeb757a 100644
--- a/base/operators.jl
+++ b/base/operators.jl
@@ -528,48 +528,36 @@ end
 macro vectorize_1arg(S,f)
     S = esc(S); f = esc(f); T = esc(:T)
     quote
-        ($f){$T<:$S}(x::AbstractArray{$T,1}) = [ ($f)(elem) for elem in x ]
-        ($f){$T<:$S}(x::AbstractArray{$T,2}) =
-            [ ($f)(x[i,j]) for i=1:size(x,1), j=1:size(x,2) ]
-        ($f){$T<:$S}(x::AbstractArray{$T}) =
-            reshape([ ($f)(y) for y in x ], size(x))
+        ($f){$T<:$S}(x::AbstractArray{$T}) = [ ($f)(elem) for elem in x ]
     end
 end
 
 macro vectorize_2arg(S,f)
     S = esc(S); f = esc(f); T1 = esc(:T1); T2 = esc(:T2)
     quote
-        ($f){$T1<:$S, $T2<:$S}(x::($T1), y::AbstractArray{$T2}) =
-            reshape([ ($f)(x, z) for z in y ], size(y))
-        ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::($T2)) =
-            reshape([ ($f)(z, y) for z in x ], size(x))
-
-        function ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::AbstractArray{$T2})
-            shp = promote_shape(size(x),size(y))
-            reshape([ ($f)(xx, yy) for (xx, yy) in zip(x, y) ], shp)
-        end
+        ($f){$T1<:$S, $T2<:$S}(x::($T1), y::AbstractArray{$T2}) = [ ($f)(x, z) for z in y ]
+        ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::($T2)) = [ ($f)(z, y) for z in x ]
+        ($f){$T1<:$S, $T2<:$S}(x::AbstractArray{$T1}, y::AbstractArray{$T2}) =
+            [ ($f)(xx, yy) for (xx, yy) in zip(x, y) ]
     end
 end
 
 # vectorized ifelse
 
 function ifelse(c::AbstractArray{Bool}, x, y)
-    reshape([ifelse(ci, x, y) for ci in c], size(c))
+    [ifelse(ci, x, y) for ci in c]
 end
 
 function ifelse(c::AbstractArray{Bool}, x::AbstractArray, y::AbstractArray)
-    shp = promote_shape(size(c), promote_shape(size(x), size(y)))
-    reshape([ifelse(c_elem, x_elem, y_elem) for (c_elem, x_elem, y_elem) in zip(c, x, y)], shp)
+    [ifelse(c_elem, x_elem, y_elem) for (c_elem, x_elem, y_elem) in zip(c, x, y)]
 end
 
 function ifelse(c::AbstractArray{Bool}, x::AbstractArray, y)
-    shp = promote_shape(size(c), size(c))
-    reshape([ifelse(c_elem, x_elem, y) for (c_elem, x_elem) in zip(c, x)], shp)
+    [ifelse(c_elem, x_elem, y) for (c_elem, x_elem) in zip(c, x)]
 end
 
 function ifelse(c::AbstractArray{Bool}, x, y::AbstractArray)
-    shp = promote_shape(size(c), size(y))
-    reshape([ifelse(c_elem, x, y_elem) for (c_elem, y_elem) in zip(c, y)], shp)
+    [ifelse(c_elem, x, y_elem) for (c_elem, y_elem) in zip(c, y)]
 end
 
 # Pair

From 9bb142373eca1256460419439ea8be29a10eed32 Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Mon, 27 Jun 2016 15:46:09 -0400
Subject: [PATCH 6/9] add comprehension filters (fixes #550) and nested loops
 (fixes #4867)

this also uses the new lowering for typed comprehensions, allowing all
comprehensions on unknown-length iterables (fixes #1457)
---
 src/julia-parser.scm | 20 +++++++---
 src/julia-syntax.scm | 90 +++++++++++++++++++++++++++++---------------
 2 files changed, 73 insertions(+), 37 deletions(-)

diff --git a/src/julia-parser.scm b/src/julia-parser.scm
index 3cd3896707f19..d29f7c04ebeb2 100644
--- a/src/julia-parser.scm
+++ b/src/julia-parser.scm
@@ -1511,22 +1511,30 @@
             (else
              (error "missing separator in array expression")))))))
 
+(define (parse-generator s first)
+  (let ((iters (parse-comma-separated-iters s)))
+    (let ((iters (if (eq? (peek-token s) 'if)
+                     (begin (take-token s)
+                            (list `(filter ,(parse-cond s) ,@iters)))
+                     iters)))
+      (if (eq? (peek-token s) 'for)
+          (begin (take-token s)
+                 `(flatten (generator ,(parse-generator s first) ,@iters)))
+          `(generator ,first ,@iters)))))
+
 (define (parse-comprehension s first closer)
-  (let ((r (parse-comma-separated-iters s)))
+  (let ((gen (parse-generator s first)))
     (if (not (eqv? (require-token s) closer))
         (error (string "expected \"" closer "\""))
         (take-token s))
-    `(comprehension ,first ,@r)))
+    `(comprehension ,gen)))
 
 (define (parse-dict-comprehension s first closer)
   (let ((c (parse-comprehension s first closer)))
-    (if (dict-literal? (cadr c))
+    (if (dict-literal? (cadr (cadr c)))
         `(dict_comprehension ,@(cdr c))
         (error "invalid dict comprehension"))))
 
-(define (parse-generator s first)
-  `(generator ,first ,@(parse-comma-separated-iters s)))
-
 (define (parse-matrix s first closer gotnewline)
   (define (fix head v) (cons head (reverse v)))
   (define (update-outer v outer)
diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm
index fa273a8fbc50a..aad862acd72b7 100644
--- a/src/julia-syntax.scm
+++ b/src/julia-syntax.scm
@@ -1526,6 +1526,26 @@
 (define (syntactic-op-to-call e)
   `(call ,(car e) ,(expand-forms (cadr e)) ,(expand-forms (caddr e))))
 
+;; wrap `expr` in a function appropriate for consuming values from given ranges
+(define (func-for-generator-ranges expr range-exprs)
+  (let* ((vars    (map cadr range-exprs))
+         (argname (if (and (length= vars 1) (symbol? (car vars)))
+                      (car vars)
+                      (gensy)))
+         (splat (cond ((eq? argname (car vars))  '())
+                      ((length= vars 1)
+                       `(,@(map (lambda (v) `(local ,v)) (lhs-vars (car vars)))
+                         (= ,(car vars) ,argname)))
+                      (else
+                       `(,@(map (lambda (v) `(local ,v)) (lhs-vars `(tuple ,@vars)))
+                         (= (tuple ,@vars) ,argname))))))
+    (if (and (null? splat)
+             (length= expr 3) (eq? (car expr) 'call)
+             (eq? (caddr expr) argname)
+             (not (expr-contains-eq argname (cadr expr))))
+        (cadr expr)  ;; eta reduce `x->f(x)` => `f`
+        `(-> ,argname (block ,@splat ,expr)))))
+
 ;; table mapping expression head to a function expanding that form
 (define expand-table
   (table
@@ -1960,52 +1980,60 @@
 
    'generator
    (lambda (e)
-     (let ((expr   (cadr e))
-           (vars   (map cadr  (cddr e)))
-           (ranges (map caddr (cddr e))))
-       (let* ((argname (if (and (length= vars 1) (symbol? (car vars)))
-                           (car vars)
-                           (gensy)))
-              (splat (cond ((eq? argname (car vars))  '())
-                           ((length= vars 1)
-                            `(,@(map (lambda (v) `(local ,v)) (lhs-vars (car vars)))
-                              (= ,(car vars) ,argname)))
-                           (else
-                            `(,@(map (lambda (v) `(local ,v)) (lhs-vars `(tuple ,@vars)))
-                              (= (tuple ,@vars) ,argname))))))
-         (expand-forms
-          `(call (top Generator)
-                 ,(if (and (null? splat)
-                           (length= expr 3) (eq? (car expr) 'call)
-                           (eq? (caddr expr) argname)
-                           (not (expr-contains-eq argname (cadr expr))))
-                      (cadr expr)  ;; eta reduce `x->f(x)` => `f`
-                      `(-> ,argname (block ,@splat ,expr)))
-                 ,(if (length= ranges 1)
+     (let* ((expr  (cadr e))
+            (filt? (eq? (car (caddr e)) 'filter))
+            (range-exprs (if filt? (cddr (caddr e)) (cddr e)))
+            (ranges (map caddr range-exprs))
+            (iter (if (length= ranges 1)
                       (car ranges)
-                      `(call (top product) ,@ranges)))))))
+                      `(call (top product) ,@ranges)))
+            (iter (if filt?
+                      `(call (top Filter)
+                             ,(func-for-generator-ranges (cadr (caddr e)) range-exprs)
+                             ,iter)
+                      iter)))
+       (expand-forms
+        `(call (top Generator)
+               ,(func-for-generator-ranges expr range-exprs)
+               ,iter))))
+
+   'flatten
+   (lambda (e) `(call (top Flatten) ,(expand-forms (cadr e))))
 
    'comprehension
    (lambda (e)
-     (if (any (lambda (x) (eq? x ':)) (cddr e))
-         (error "comprehension syntax with `:` ranges has been removed"))
-     (expand-forms `(call (top collect) (generator ,(cadr e) ,@(cddr e)))))
+     (if (length> e 2)
+         ;; backwards compat for macros that generate :comprehension exprs
+         (expand-forms `(comprehension (generator ,@(cdr e))))
+         (begin (if (and (eq? (caadr e) 'generator)
+                         (any (lambda (x) (eq? x ':)) (cddr (cadr e))))
+                    (error "comprehension syntax with `:` ranges has been removed"))
+                (expand-forms `(call (top collect) ,(cadr e))))))
 
    'typed_comprehension
    (lambda (e)
-     (if (any (lambda (x) (eq? x ':)) (cdddr e))
-         (error "comprehension syntax with `:` ranges has been removed"))
-     (expand-forms (lower-comprehension (cadr e) (caddr e) (cdddr e))))
+     (expand-forms
+      (or (and (eq? (caaddr e) 'generator)
+               (let ((ranges (cddr (caddr e))))
+                 (if (any (lambda (x) (eq? x ':)) ranges)
+                     (error "comprehension syntax with `:` ranges has been removed"))
+                 (and (every (lambda (x) (and (pair? x) (eq? (car x) '=)
+                                              (pair? (caddr x)) (eq? (car (caddr x)) ':)))
+                             ranges)
+                      ;; TODO: this is a hack to lower simple comprehensions to loops very
+                      ;; early, to greatly reduce the # of functions and load on the compiler
+                      (lower-comprehension (cadr e) (cadr (caddr e)) ranges))))
+          `(call (top collect) ,(cadr e) ,(caddr e)))))
 
    'dict_comprehension
    (lambda (e)
      (syntax-deprecation #f "[a=>b for (a,b) in c]" "Dict(a=>b for (a,b) in c)")
-     (expand-forms `(call (top Dict) (generator ,(cadr e) ,@(cddr e)))))
+     (expand-forms `(call (top Dict) ,(cadr e))))
 
    'typed_dict_comprehension
    (lambda (e) (expand-forms
                 `(call (call (core apply_type) (top Dict) ,@(cdr (cadr e)))
-                       (generator ,(caddr e) ,@(cdddr e)))))))
+                       ,(caddr e))))))
 
 (define (lower-comprehension atype expr ranges)
   (let ((result    (make-ssavalue))

From 74f5c034a160e036b7074b0e1e3e557a26df0148 Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Thu, 23 Jun 2016 21:25:15 -0400
Subject: [PATCH 7/9] doc/NEWS updates for comprehension changes

---
 NEWS.md                              | 10 +++++++
 doc/manual/arrays.rst                | 40 +++++++++++++++++++++-------
 doc/manual/variables-and-scoping.rst |  6 ++---
 3 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/NEWS.md b/NEWS.md
index d332f885826bc..ed0d84c1a1d73 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -7,6 +7,9 @@ New language features
   * Generator expressions, e.g. `f(i) for i in 1:n` ([#4470]). This returns an iterator
     that computes the specified values on demand.
 
+  * Generators and comprehensions support filtering using `if` ([#550]) and nested
+    iteration using multiple `for` keywords ([#4867]).
+
   * Broadcasting syntax: ``f.(args...)`` is equivalent to ``broadcast(f, args...)`` ([#15032]).
 
   * Macro expander functions are now generic, so macros can have multiple definitions
@@ -69,6 +72,13 @@ Language changes
   * The built-in `NTuple` type has been removed; `NTuple{N,T}` is now
     implemented internally as `Tuple{Vararg{T,N}}` ([#11242]).
 
+  * Array comprehensions preserve the dimensions of the input ranges. For example,
+    `[ 2x for x in A]` will have the same dimensions as `A`.
+
+  * The result type of an array comprehension depends only on the types of elements
+    computed, instead of using type inference ([#7258]). If the result is empty, then
+    type inference is still used to determine the element type.
+
 Command-line option changes
 ---------------------------
 
diff --git a/doc/manual/arrays.rst b/doc/manual/arrays.rst
index 007483ae3c8db..84207da57dbb7 100644
--- a/doc/manual/arrays.rst
+++ b/doc/manual/arrays.rst
@@ -185,7 +185,7 @@ and its left and right neighbor along a 1-d grid. :
 
 .. doctest:: array-rand
 
-    julia> const x = rand(8)
+    julia> x = rand(8)
     8-element Array{Float64,1}:
      0.843025
      0.869052
@@ -205,16 +205,11 @@ and its left and right neighbor along a 1-d grid. :
      0.8446
      0.656511
 
-.. note:: In the above example, ``x`` is declared as constant because type
-  inference in Julia does not work as well on non-constant global
-  variables.
+The resulting array type depends on the types of the computed elements.
+In order to control the type explicitly, a type can be prepended to the comprehension.
+For example, we could have requested the result in single precision by writing::
 
-The resulting array type is inferred from the expression; in order to control
-the type explicitly, the type can be prepended to the comprehension. For example,
-in the above example we could have avoided declaring ``x`` as constant, and ensured
-that the result is of type ``Float64`` by writing::
-
-    Float64[ 0.25*x[i-1] + 0.5*x[i] + 0.25*x[i+1] for i=2:length(x)-1 ]
+    Float32[ 0.25*x[i-1] + 0.5*x[i] + 0.25*x[i+1] for i=2:length(x)-1 ]
 
 .. _man-generator-expressions:
 
@@ -248,6 +243,31 @@ parentheses lets us add a third argument to ``map``:
      (0.5,1)       (0.333333,3)
      (0.333333,2)  (0.25,4)
 
+Ranges in generators and comprehensions can depend on previous ranges by writing
+multiple ``for`` keywords:
+
+.. doctest::
+
+    julia> [(i,j) for i=1:3 for j=1:i]
+    6-element Array{Tuple{Int64,Int64},1}:
+     (1,1)
+     (2,1)
+     (2,2)
+     (3,1)
+     (3,2)
+     (3,3)
+
+In such cases, the result is always 1-d.
+
+Generated values can be filtered using the ``if`` keyword:
+
+.. doctest::
+
+    julia> [(i,j) for i=1:3 for j=1:i if i+j == 4]
+    2-element Array{Tuple{Int64,Int64},1}:
+     (2,2)
+     (3,1)
+
 .. _man-array-indexing:
 
 Indexing
diff --git a/doc/manual/variables-and-scoping.rst b/doc/manual/variables-and-scoping.rst
index 7840eb03ae636..157110fcec7ba 100644
--- a/doc/manual/variables-and-scoping.rst
+++ b/doc/manual/variables-and-scoping.rst
@@ -31,7 +31,7 @@ introducing scope blocks are:
 +================================+==================================================================================+
 | :ref:`global <man-global>`     | module, baremodule, at interactive prompt (REPL)                                 |
 +--------------------------------+------------------------------+---------------------------------------------------+
-| :ref:`local <man-local-scope>` | :ref:`soft <man-soft-scope>` | for, while, list-comprehensions,                  |
+| :ref:`local <man-local-scope>` | :ref:`soft <man-soft-scope>` | for, while, comprehensions,                       |
 |                                |                              | try-catch-finally, let                            |
 |                                +------------------------------+---------------------------------------------------+
 |                                | :ref:`hard <man-hard-scope>` | functions (either syntax, anonymous & do-blocks), |
@@ -188,9 +188,9 @@ Soft Local Scope
   ``local``.
 
 Soft local scopes are introduced by for-loops, while-loops,
-list-comprehensions, try-catch-finally-blocks, and let-blocks.  There
+comprehensions, try-catch-finally-blocks, and let-blocks.  There
 are some extra rules for :ref:`let-blocks <man-let-blocks>` and for
-:ref:`for-loops and list-comprehensions <man-for-loops-scope>`.
+:ref:`for-loops and comprehensions <man-for-loops-scope>`.
 
 In the following example the ``x`` and ``y`` refer always to the same
 variables as the soft local scope inherits both read and write

From 4fe43d05d2eb79b6acd9a5d3279bcfaa1f7e122f Mon Sep 17 00:00:00 2001
From: Jacob Quinn <quinn.jacobd@gmail.com>
Date: Fri, 24 Jun 2016 21:13:24 -0400
Subject: [PATCH 8/9] Tests and documentation for generators with guards and
 nesting.

---
 base/generator.jl  |  5 +++--
 test/functional.jl | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/base/generator.jl b/base/generator.jl
index 732aefca93903..75625c2c40525 100644
--- a/base/generator.jl
+++ b/base/generator.jl
@@ -5,8 +5,9 @@
 
 Given a function `f` and an iterator `iter`, construct an iterator that yields
 the values of `f` applied to the elements of `iter`.
-The syntax `f(x) for x in iter` is syntax for constructing an instance of this
-type.
+The syntax `f(x) [if cond(x)::Bool] for x in iter` is syntax for constructing an instance of this
+type. The `[if cond(x)::Bool]` expression is optional and acts as a "guard", effectively
+filtering out values where the condition is false.
 """
 immutable Generator{I,F}
     f::F
diff --git a/test/functional.jl b/test/functional.jl
index e8f890f7ede7c..b495ddb715847 100644
--- a/test/functional.jl
+++ b/test/functional.jl
@@ -437,6 +437,43 @@ let i = 1
     @test i == 1
 end
 
+# generators and guards
+
+let gen = (x for x in 1:10)
+    @test gen.iter == 1:10
+    @test gen.f(first(1:10)) == next(gen, start(gen))[1]
+    for (a,b) in zip(1:10,gen)
+        @test a == b
+    end
+end
+
+let gen = (x * y for x in 1:10, y in 1:10)
+    @test collect(gen) == collect(1:10) .* collect(1:10)'
+    @test first(gen) == 1
+    @test collect(gen)[1:10] == collect(1:10)
+end
+
+let gen = Base.Generator(+, 1:10, 1:10, 1:10)
+    @test first(gen) == 3
+    @test collect(gen) == collect(3:3:30)
+end
+
+let gen = (x for x in 1:10 if x % 2 == 0), gen2 = Filter(x->x % 2 == 0, x for x in 1:10)
+    @test collect(gen) == collect(gen2)
+    @test collect(gen) == collect(2:2:10)
+end
+
+let gen = ((x,y) for x in 1:10, y in 1:10 if x % 2 == 0 && y % 2 == 0),
+    gen2 = Filter(x->x[1] % 2 == 0 && x[2] % 2 == 0, (x,y) for x in 1:10, y in 1:10)
+    @test collect(gen) == collect(gen2)
+end
+
+# generators with nested loops (#4867)
+
+@test [(i,j) for i=1:3 for j=1:i] == [(1,1), (2,1), (2,2), (3,1), (3,2), (3,3)]
+
+@test [(i,j) for i=1:3 for j=1:i if j>1] == [(2,2), (3,2), (3,3)]
+
 # partition(c, n)
 let v = collect(Base.partition([1,2,3,4,5], 1))
     @test all(i->v[i][1] == i, v)

From b363cc70ee6f09bb4472944f8690283793c6052a Mon Sep 17 00:00:00 2001
From: Jeff Bezanson <jeff.bezanson@gmail.com>
Date: Wed, 6 Jul 2016 00:20:20 -0400
Subject: [PATCH 9/9] shorten call chain for `collect(::Generator)`

---
 base/array.jl | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/base/array.jl b/base/array.jl
index 81e0cc2866fc5..2a19d201cabda 100644
--- a/base/array.jl
+++ b/base/array.jl
@@ -254,6 +254,24 @@ else
 end
 _default_eltype{I,T}(::Type{Generator{I,Type{T}}}) = T
 
+_array_for(T, itr, ::HasLength) = Array(T, Int(length(itr)::Integer))
+_array_for(T, itr, ::HasShape) = Array(T, convert(Dims,size(itr)))
+
+function collect(itr::Generator)
+    isz = iteratorsize(itr.iter)
+    et = _default_eltype(typeof(itr))
+    if isa(isz, SizeUnknown)
+        return grow_to!(Array(et, 0), itr)
+    else
+        st = start(itr)
+        if done(itr,st)
+            return _array_for(et, itr.iter, isz)
+        end
+        v1, st = next(itr, st)
+        collect_to_with_first!(_array_for(typeof(v1), itr.iter, isz), v1, itr, st)
+    end
+end
+
 _collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) =
     grow_to!(_similar_for(c, _default_eltype(typeof(itr)), itr, isz), itr)